diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 19e09c969..6389a50df 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,20 +1,21 @@ FROM freqtradeorg/freqtrade:develop +USER root # Install dependencies COPY requirements-dev.txt /freqtrade/ + RUN apt-get update \ - && apt-get -y install git mercurial sudo vim \ + && apt-get -y install git mercurial sudo vim build-essential \ && apt-get clean \ - && pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \ - && useradd -u 1000 -U -m ftuser \ && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \ && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \ && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \ - && mv /root/.local /home/ftuser/.local/ \ && chown ftuser:ftuser -R /home/ftuser/.local/ \ && chown ftuser: -R /home/ftuser/ USER ftuser +RUN pip install --user autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir + # Empty the ENTRYPOINT to allow all commands ENTRYPOINT [] diff --git a/.dockerignore b/.dockerignore index 09f4c9f0c..abc5b82f0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,9 @@ .git .gitignore Dockerfile +Dockerfile.armhf .dockerignore -config.json* -*.sqlite +docker/ .coveragerc .eggs .github @@ -13,4 +13,13 @@ CONTRIBUTING.md MANIFEST.in README.md freqtrade.service +freqtrade.egg-info + +config.json* +*.sqlite user_data +*.log + +.vscode +.mypy_cache +.ipynb_checkpoints diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..81f6f422e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.py eol=lf +*.sh eol=lf +*.ps1 eol=crlf diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..5014de46a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,6 @@ +--- +blank_issues_enabled: false +contact_links: + - name: Discord Server + url: https://discord.gg/p7nuUNVfP7 + about: Ask a question or get community support from our Discord server diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f294347a..42959c3b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,7 @@ jobs: COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu run: | # Allow failure for coveralls - coveralls -v || true + coveralls || true - name: Backtesting run: | @@ -102,7 +102,7 @@ jobs: mypy freqtrade scripts - name: Slack Notification - uses: homoluctus/slatify@v1.8.0 + uses: lazy-actions/slatify@v3.0.0 if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: type: ${{ job.status }} @@ -148,6 +148,7 @@ jobs: - name: Installation - macOS run: | + brew update brew install hdf5 c-blosc python -m pip install --upgrade pip export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH @@ -194,7 +195,7 @@ jobs: mypy freqtrade scripts - name: Slack Notification - uses: homoluctus/slatify@v1.8.0 + uses: lazy-actions/slatify@v3.0.0 if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: type: ${{ job.status }} @@ -257,7 +258,7 @@ jobs: mypy freqtrade scripts - name: Slack Notification - uses: homoluctus/slatify@v1.8.0 + uses: lazy-actions/slatify@v3.0.0 if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: type: ${{ job.status }} @@ -288,7 +289,7 @@ jobs: mkdocs build - name: Slack Notification - uses: homoluctus/slatify@v1.8.0 + uses: lazy-actions/slatify@v3.0.0 if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: type: ${{ job.status }} @@ -300,7 +301,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Cleanup previous runs on this branch - uses: rokroskar/workflow-run-cleanup-action@v0.2.2 + uses: rokroskar/workflow-run-cleanup-action@v0.3.3 if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -310,9 +311,18 @@ jobs: needs: [ build_linux, build_macos, build_windows, docs_check ] runs-on: ubuntu-20.04 steps: + + - name: Check user permission + id: check + uses: scherermichael-oss/action-has-permission@1.0.6 + with: + required-permission: write + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Slack Notification - uses: homoluctus/slatify@v1.8.0 - if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) + uses: lazy-actions/slatify@v3.0.0 + if: always() && steps.check.outputs.has-permission && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: type: ${{ job.status }} job_name: '*Freqtrade CI*' @@ -364,13 +374,6 @@ jobs: run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Build and test and push docker image - env: - IMAGE_NAME: freqtradeorg/freqtrade - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} - run: | - build_helpers/publish_docker.sh - # We need docker experimental to pull the ARM image. - name: Switch docker to experimental run: | @@ -389,16 +392,16 @@ jobs: - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} - - name: Build Raspberry docker image + - name: Build and test and push docker images env: IMAGE_NAME: freqtradeorg/freqtrade - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}_pi + BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} run: | - build_helpers/publish_docker_pi.sh + build_helpers/publish_docker_multi.sh - name: Slack Notification - uses: homoluctus/slatify@v1.8.0 + uses: lazy-actions/slatify@v3.0.0 if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) with: type: ${{ job.status }} diff --git a/.travis.yml b/.travis.yml index 03a8df49b..4535c44cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,12 +46,6 @@ jobs: - script: mypy freqtrade scripts name: mypy - # - stage: docker - # if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron)) - # script: - # - build_helpers/publish_docker.sh - # name: "Build and test and push docker image" - notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afa41ed33..040cf3e98 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Few pointers for contributions: - New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR. - PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished). -If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. +If you are unsure, discuss the feature on our [discord server](https://discord.gg/p7nuUNVfP7), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. ## Getting started diff --git a/Dockerfile b/Dockerfile index 8d4f0ebe6..f2d7c8a40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,31 @@ -FROM python:3.9.1-slim-buster as base +FROM python:3.9.5-slim-buster as base # Setup env ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONFAULTHANDLER 1 -ENV PATH=/root/.local/bin:$PATH +ENV PATH=/home/ftuser/.local/bin:$PATH +ENV FT_APP_ENV="docker" # Prepare environment -RUN mkdir /freqtrade +RUN mkdir /freqtrade \ + && apt-get update \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ + && apt-get clean \ + && useradd -u 1000 -G sudo -U -m ftuser \ + && chown ftuser:ftuser /freqtrade \ + # Allow sudoers + && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers + WORKDIR /freqtrade # Install dependencies FROM base as python-deps -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev git \ - && apt-get clean \ - && pip install --upgrade pip +RUN apt-get update \ + && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \ + && apt-get clean \ + && pip install --upgrade pip # Install TA-lib COPY build_helpers/* /tmp/ @@ -24,7 +33,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies -COPY requirements.txt requirements-hyperopt.txt /freqtrade/ +COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/ +USER ftuser RUN pip install --user --no-cache-dir numpy \ && pip install --user --no-cache-dir -r requirements-hyperopt.txt @@ -33,13 +43,13 @@ FROM base as runtime-image COPY --from=python-deps /usr/local/lib /usr/local/lib ENV LD_LIBRARY_PATH /usr/local/lib -COPY --from=python-deps /root/.local /root/.local - - +COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local +USER ftuser # Install and execute -COPY . /freqtrade/ -RUN pip install -e . --no-cache-dir \ +COPY --chown=ftuser:ftuser . /freqtrade/ + +RUN pip install -e . --user --no-cache-dir --no-build-isolation \ && mkdir /freqtrade/user_data/ \ && freqtrade install-ui diff --git a/README.md b/README.md index bb136d7f2..c665508dd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Freqtrade +# ![freqtrade](https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docs/assets/freqtrade_poweredby.svg) [![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/) [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) @@ -22,12 +22,21 @@ expect. We strongly recommend you to have coding and Python knowledge. Do not hesitate to read the source code and understand the mechanism of this bot. -## Exchange marketplaces supported +## Supported Exchange marketplaces + +Please read the [exchange specific notes](docs/exchanges.md) to learn about eventual, special configurations needed for each exchange. - [X] [Bittrex](https://bittrex.com/) - [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#blacklists)) - [X] [Kraken](https://kraken.com/) -- [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ +- [X] [FTX](https://ftx.com) +- [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ + +### Community tested + +Exchanges confirmed working by the community: + +- [X] [Bitvavo](https://bitvavo.com/) ## Documentation @@ -114,7 +123,7 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor - `/stop`: Stops the trader. - `/stopbuy`: Stop entering new trades. - `/status |[table]`: Lists all or specific open trades. -- `/profit`: Lists cumulative profit from all finished trades +- `/profit []`: Lists cumulative profit from all finished trades, over the last n days. - `/forcesell |all`: Instantly sells the given trade (Ignoring `minimum_roi`). - `/performance`: Show performance of each finished trade grouped by pair - `/balance`: Show account balance per currency. @@ -136,16 +145,16 @@ The project is currently setup in two main branches: For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. -Please check out our [discord server](https://discord.gg/MA9v74M). +Please check out our [discord server](https://discord.gg/p7nuUNVfP7). -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA). +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) If you discover a bug in the bot, please [search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) first. If it hasn't been reported, please -[create a new issue](https://github.com/freqtrade/freqtrade/issues/new) and +[create a new issue](https://github.com/freqtrade/freqtrade/issues/new/choose) and ensure you follow the template guide so that our team can assist you as quickly as possible. @@ -154,7 +163,7 @@ quickly as possible. Have you a great idea to improve the bot you want to share? Please, first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement). If it hasn't been requested, please -[create a new request](https://github.com/freqtrade/freqtrade/issues/new) +[create a new request](https://github.com/freqtrade/freqtrade/issues/new/choose) and ensure you follow the template guide so that it does not get lost in the bug reports. @@ -169,7 +178,7 @@ to understand the requirements before sending your pull-requests. Coding is not a necessity to contribute - maybe start with improving our documentation? Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. -**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/MA9v74M) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/p7nuUNVfP7) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. **Important:** Always create your PR against the `develop` branch, not `stable`. diff --git a/build_helpers/TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl deleted file mode 100644 index 5adbda8a7..000000000 Binary files a/build_helpers/TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.19-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.19-cp38-cp38-win_amd64.whl deleted file mode 100644 index b652c7ee0..000000000 Binary files a/build_helpers/TA_Lib-0.4.19-cp38-cp38-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl new file mode 100644 index 000000000..b4eee9c47 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.20-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.20-cp38-cp38-win_amd64.whl new file mode 100644 index 000000000..de7393fde Binary files /dev/null and b/build_helpers/TA_Lib-0.4.20-cp38-cp38-win_amd64.whl differ diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index cb86e5f64..dd87cf105 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -8,10 +8,13 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then tar zxvf ta-lib-0.4.0-src.tar.gz cd ta-lib \ && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ + && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess \ + && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \ && ./configure --prefix=${INSTALL_LOC}/ \ - && make \ + && make -j$(nproc) \ && which sudo && sudo make install || make install \ && cd .. else echo "TA-lib already installed, skipping installation" fi +# && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 index 5747db335..d2a27dd26 100644 --- a/build_helpers/install_windows.ps1 +++ b/build_helpers/install_windows.ps1 @@ -1,16 +1,15 @@ # Downloads don't work automatically, since the URL is regenerated via javascript. # Downloaded from https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib -# Invoke-WebRequest -Uri "https://download.lfd.uci.edu/pythonlibs/xxxxxxx/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" -OutFile "TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" python -m pip install --upgrade pip $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" if ($pyv -eq '3.7') { - pip install build_helpers\TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl } if ($pyv -eq '3.8') { - pip install build_helpers\TA_Lib-0.4.19-cp38-cp38-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.20-cp38-cp38-win_amd64.whl } pip install -r requirements-dev.txt diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker_multi.sh similarity index 52% rename from build_helpers/publish_docker.sh rename to build_helpers/publish_docker_multi.sh index 9bc1aa0a6..a6b06ce7d 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker_multi.sh @@ -1,21 +1,48 @@ #!/bin/sh +# The below assumes a correctly setup docker buildx environment + # Replace / with _ to create a valid tag TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") TAG_PLOT=${TAG}_plot +TAG_PI="${TAG}_pi" + +PI_PLATFORM="linux/arm/v7" echo "Running for ${TAG}" +CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache # Add commit and commit_message to docker container echo "${GITHUB_SHA}" > freqtrade_commit if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" + # Build regular image docker build -t freqtrade:${TAG} . + # Build PI image + docker buildx build \ + --cache-to=type=registry,ref=${CACHE_TAG} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} --push . else echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Pull last build to avoid rebuilding the whole image + # Build regular image docker pull ${IMAGE_NAME}:${TAG} docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . + + # Pull last build to avoid rebuilding the whole image + # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} + docker buildx build \ + --cache-from=type=registry,ref=${CACHE_TAG} \ + --cache-to=type=registry,ref=${CACHE_TAG} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} --push . +fi + +if [ $? -ne 0 ]; then + echo "failed building multiarch images" + return 1 fi # Tag image for upload and next build step docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG @@ -24,11 +51,6 @@ docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t fre docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi - # Run backtest docker run --rm -v $(pwd)/config_bittrex.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy @@ -37,21 +59,29 @@ if [ $? -ne 0 ]; then return 1 fi -if [ $? -ne 0 ]; then - echo "failed tagging image" - return 1 -fi - -# Tag as latest for develop builds -if [ "${TAG}" = "develop" ]; then - docker tag freqtrade:$TAG ${IMAGE_NAME}:latest -fi - -# Show all available images docker images docker push ${IMAGE_NAME} +docker push ${IMAGE_NAME}:$TAG_PLOT +docker push ${IMAGE_NAME}:$TAG + +# Create multiarch image +# Make sure that all images contained here are pushed to github first. +# Otherwise installation might fail. + +docker manifest create freqtradeorg/freqtrade:${TAG} ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} +docker manifest push freqtradeorg/freqtrade:${TAG} + +# Tag as latest for develop builds +if [ "${TAG}" = "develop" ]; then + docker manifest create freqtradeorg/freqtrade:latest ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} + docker manifest push freqtradeorg/freqtrade:latest +fi + + +docker images + if [ $? -ne 0 ]; then - echo "failed pushing repo" + echo "failed building image" return 1 fi diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh deleted file mode 100755 index 060b1deaf..000000000 --- a/build_helpers/publish_docker_pi.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh - -# The below assumes a correctly setup docker buildx environment - -# Replace / with _ to create a valid tag -TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") -PI_PLATFORM="linux/arm/v7" -echo "Running for ${TAG}" -CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache - -# Add commit and commit_message to docker container -echo "${GITHUB_SHA}" > freqtrade_commit - -if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then - echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" - docker buildx build \ - --cache-to=type=registry,ref=${CACHE_TAG} \ - -f Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG} --push . -else - echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Pull last build to avoid rebuilding the whole image - # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} - docker buildx build \ - --cache-from=type=registry,ref=${CACHE_TAG} \ - --cache-to=type=registry,ref=${CACHE_TAG} \ - -f Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG} --push . -fi - -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi diff --git a/config_bittrex.json.example b/config_bittrex.json.example index 0f0bbec4b..172cfcfc3 100644 --- a/config_bittrex.json.example +++ b/config_bittrex.json.example @@ -41,13 +41,13 @@ "ETH/BTC", "LTC/BTC", "ETC/BTC", - "DASH/BTC", - "ZEC/BTC", + "RVN/BTC", + "CRO/BTC", "XLM/BTC", "XRP/BTC", "TRX/BTC", "ADA/BTC", - "XMR/BTC" + "DOT/BTC" ], "pair_blacklist": [ "DOGE/BTC" diff --git a/config_ftx.json.example b/config_ftx.json.example new file mode 100644 index 000000000..facd54b25 --- /dev/null +++ b/config_ftx.json.example @@ -0,0 +1,99 @@ +{ + "max_open_trades": 3, + "stake_currency": "USD", + "stake_amount": 50, + "tradable_balance_ratio": 0.99, + "fiat_display_currency": "USD", + "timeframe": "5m", + "dry_run": true, + "cancel_open_orders_on_exit": false, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, + "bid_strategy": { + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy": { + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 1, + "use_sell_signal": true, + "sell_profit_only": false, + "ignore_roi_if_buy_signal": false + }, + "exchange": { + "name": "ftx", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 50 + }, + "pair_whitelist": [ + "BTC/USD", + "ETH/USD", + "BNB/USD", + "USDT/USD", + "LTC/USD", + "SRM/USD", + "SXP/USD", + "XRP/USD", + "DOGE/USD", + "1INCH/USD", + "CHZ/USD", + "MATIC/USD", + "LINK/USD", + "OXY/USD", + "SUSHI/USD" + ], + "pair_blacklist": [ + "FTT/USD" + ] + }, + "pairlists": [ + {"method": "StaticPairList"} + ], + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 7, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, + "telegram": { + "enabled": false, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id" + }, + "api_server": { + "enabled": false, + "listen_ip_address": "127.0.0.1", + "listen_port": 8080, + "verbosity": "error", + "jwt_secret_key": "somethingrandom", + "CORS_origins": [], + "username": "freqtrader", + "password": "SuperSecurePassword" + }, + "bot_name": "freqtrade", + "initial_state": "running", + "forcebuy_enable": false, + "internals": { + "process_throttle_secs": 5 + } +} diff --git a/config_full.json.example b/config_full.json.example index 6593750b4..6aeb756f3 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -23,7 +23,8 @@ "stoploss": -0.10, "unfilledtimeout": { "buy": 10, - "sell": 30 + "sell": 30, + "unit": "minutes" }, "bid_strategy": { "price_side": "bid", @@ -49,6 +50,8 @@ "buy": "limit", "sell": "limit", "emergencysell": "market", + "forcesell": "market", + "forcebuy": "market", "stoploss": "market", "stoploss_on_exchange": false, "stoploss_on_exchange_interval": 60 @@ -111,7 +114,7 @@ "password": "", "ccxt_config": {"enableRateLimit": true}, "ccxt_async_config": { - "enableRateLimit": false, + "enableRateLimit": true, "rateLimit": 500, "aiohttp_trust_env": false }, @@ -161,7 +164,18 @@ "warning": "on", "startup": "on", "buy": "on", - "sell": "on", + "buy_fill": "on", + "sell": { + "roi": "off", + "emergency_sell": "off", + "force_sell": "off", + "sell_signal": "off", + "trailing_stop_loss": "off", + "stop_loss": "off", + "stoploss_on_exchange": "off", + "custom_sell": "off" + }, + "sell_fill": "on", "buy_cancel": "on", "sell_cancel": "on" } diff --git a/docker-compose.yml b/docker-compose.yml index 1f63059f0..80e194ab2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: # Build step - only needed when additional dependencies are needed # build: # context: . - # dockerfile: "./docker/Dockerfile.technical" + # dockerfile: "./docker/Dockerfile.custom" restart: unless-stopped container_name: freqtrade volumes: diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 new file mode 100644 index 000000000..e5d3f0ee9 --- /dev/null +++ b/docker/Dockerfile.aarch64 @@ -0,0 +1,58 @@ +FROM --platform=linux/arm64/v8 python:3.9.4-slim-buster as base + +# Setup env +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONFAULTHANDLER 1 +ENV PATH=/home/ftuser/.local/bin:$PATH +ENV FT_APP_ENV="docker" + +# Prepare environment +RUN mkdir /freqtrade \ + && apt-get update \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ + && apt-get clean \ + && useradd -u 1000 -G sudo -U -m ftuser \ + && chown ftuser:ftuser /freqtrade \ + # Allow sudoers + && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers + +WORKDIR /freqtrade + +# Install dependencies +FROM base as python-deps +RUN apt-get update \ + && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \ + && apt-get clean \ + && pip install --upgrade pip + +# Install TA-lib +COPY build_helpers/* /tmp/ +RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* +ENV LD_LIBRARY_PATH /usr/local/lib + +# Install dependencies +COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/ +USER ftuser +RUN pip install --user --no-cache-dir numpy \ + && pip install --user --no-cache-dir -r requirements-hyperopt.txt + +# Copy dependencies to runtime-image +FROM base as runtime-image +COPY --from=python-deps /usr/local/lib /usr/local/lib +ENV LD_LIBRARY_PATH /usr/local/lib + +COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local + +USER ftuser +# Install and execute +COPY --chown=ftuser:ftuser . /freqtrade/ + +RUN pip install -e . --user --no-cache-dir --no-build-isolation\ + && mkdir /freqtrade/user_data/ \ + && freqtrade install-ui + +ENTRYPOINT ["freqtrade"] +# Default to trade mode +CMD [ "trade" ] diff --git a/Dockerfile.armhf b/docker/Dockerfile.armhf similarity index 52% rename from Dockerfile.armhf rename to docker/Dockerfile.armhf index f938ec457..f9827774e 100644 --- a/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,23 +1,29 @@ -FROM --platform=linux/arm/v7 python:3.7.9-slim-buster as base +FROM python:3.7.10-slim-buster as base # Setup env ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONFAULTHANDLER 1 -ENV PATH=/root/.local/bin:$PATH +ENV PATH=/home/ftuser/.local/bin:$PATH +ENV FT_APP_ENV="docker" # Prepare environment -RUN mkdir /freqtrade -WORKDIR /freqtrade +RUN mkdir /freqtrade \ + && apt-get update \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev \ + && apt-get clean \ + && useradd -u 1000 -G sudo -U -m ftuser \ + && chown ftuser:ftuser /freqtrade \ + # Allow sudoers + && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers -RUN apt-get update \ - && apt-get -y install libatlas3-base curl sqlite3 \ - && apt-get clean +WORKDIR /freqtrade # Install dependencies FROM base as python-deps -RUN apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \ +RUN apt-get update \ + && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 pkg-config cmake gcc \ && apt-get clean \ && pip install --upgrade pip \ && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf @@ -28,7 +34,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies -COPY requirements.txt /freqtrade/ +COPY --chown=ftuser:ftuser requirements.txt /freqtrade/ +USER ftuser RUN pip install --user --no-cache-dir numpy \ && pip install --user --no-cache-dir -r requirements.txt @@ -37,11 +44,14 @@ FROM base as runtime-image COPY --from=python-deps /usr/local/lib /usr/local/lib ENV LD_LIBRARY_PATH /usr/local/lib -COPY --from=python-deps /root/.local /root/.local +COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local +USER ftuser # Install and execute -COPY . /freqtrade/ -RUN pip install -e . --no-cache-dir \ +COPY --chown=ftuser:ftuser . /freqtrade/ + +RUN pip install -e . --user --no-cache-dir --no-build-isolation\ + && mkdir /freqtrade/user_data/ \ && freqtrade install-ui ENTRYPOINT ["freqtrade"] diff --git a/docker/Dockerfile.custom b/docker/Dockerfile.custom new file mode 100644 index 000000000..3b55fcb0e --- /dev/null +++ b/docker/Dockerfile.custom @@ -0,0 +1,10 @@ +FROM freqtradeorg/freqtrade:develop + +# Switch user to root if you must install something from apt +# Don't forget to switch the user back below! +# USER root + +# The below dependency - pyti - serves as an example. Please use whatever you need! +RUN pip install --user pyti + +# USER ftuser diff --git a/docker/Dockerfile.develop b/docker/Dockerfile.develop index cb49984e2..7c580f234 100644 --- a/docker/Dockerfile.develop +++ b/docker/Dockerfile.develop @@ -3,8 +3,8 @@ FROM freqtradeorg/freqtrade:develop # Install dependencies COPY requirements-dev.txt /freqtrade/ -RUN pip install numpy --no-cache-dir \ - && pip install -r requirements-dev.txt --no-cache-dir +RUN pip install numpy --user --no-cache-dir \ + && pip install -r requirements-dev.txt --user --no-cache-dir # Empty the ENTRYPOINT to allow all commands ENTRYPOINT [] diff --git a/docker/Dockerfile.jupyter b/docker/Dockerfile.jupyter index b7499eeef..7d603c667 100644 --- a/docker/Dockerfile.jupyter +++ b/docker/Dockerfile.jupyter @@ -1,7 +1,7 @@ FROM freqtradeorg/freqtrade:develop_plot -RUN pip install jupyterlab --no-cache-dir +RUN pip install jupyterlab --user --no-cache-dir # Empty the ENTRYPOINT to allow all commands ENTRYPOINT [] diff --git a/docker/Dockerfile.plot b/docker/Dockerfile.plot index 40bc72bc5..d2fc3618a 100644 --- a/docker/Dockerfile.plot +++ b/docker/Dockerfile.plot @@ -4,4 +4,4 @@ FROM freqtradeorg/freqtrade:${sourceimage} # Install dependencies COPY requirements-plot.txt /freqtrade/ -RUN pip install -r requirements-plot.txt --no-cache-dir +RUN pip install -r requirements-plot.txt --user --no-cache-dir diff --git a/docker/Dockerfile.technical b/docker/Dockerfile.technical deleted file mode 100644 index 9431e72d0..000000000 --- a/docker/Dockerfile.technical +++ /dev/null @@ -1,6 +0,0 @@ -FROM freqtradeorg/freqtrade:develop - -RUN apt-get update \ - && apt-get -y install git \ - && apt-get clean \ - && pip install git+https://github.com/freqtrade/technical diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index bead18038..35fd3de4a 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -4,34 +4,6 @@ This page explains some advanced Hyperopt topics that may require higher coding skills and Python knowledge than creation of an ordinal hyperoptimization class. -## Derived hyperopt classes - -Custom hyperop classes can be derived in the same way [it can be done for strategies](strategy-customization.md#derived-strategies). - -Applying to hyperoptimization, as an example, you may override how dimensions are defined in your optimization hyperspace: - -```python -class MyAwesomeHyperOpt(IHyperOpt): - ... - # Uses default stoploss dimension - -class MyAwesomeHyperOpt2(MyAwesomeHyperOpt): - @staticmethod - def stoploss_space() -> List[Dimension]: - # Override boundaries for stoploss - return [ - Real(-0.33, -0.01, name='stoploss'), - ] -``` - -and then quickly switch between hyperopt classes, running optimization process with hyperopt class you need in each particular case: - -``` -$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy ... -or -$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt2 --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy ... -``` - ## Creating and using a custom loss function To use a custom loss function class, make sure that the function `hyperopt_loss_function` is defined in your custom hyperopt loss class. @@ -40,6 +12,11 @@ For the sample below, you then need to add the command line parameter `--hyperop A sample of this can be found below, which is identical to the Default Hyperopt loss implementation. A full sample can be found in [userdata/hyperopts](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_loss.py). ``` python +from datetime import datetime +from typing import Dict + +from pandas import DataFrame + from freqtrade.optimize.hyperopt import IHyperOptLoss TARGET_TRADES = 600 @@ -54,6 +31,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, + config: Dict, processed: Dict[str, DataFrame], *args, **kwargs) -> float: """ Objective function, returns smaller number for better results @@ -81,6 +59,8 @@ Currently, the arguments are: * `trade_count`: Amount of trades (identical to `len(results)`) * `min_date`: Start date of the timerange used * `min_date`: End date of the timerange used +* `config`: Config object used (Note: Not all strategy-related parameters will be updated here if they are part of a hyperopt space). +* `processed`: Dict of Dataframes with the pair as keys containing the data used for backtesting. This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. @@ -89,3 +69,315 @@ This function needs to return a floating point number (`float`). Smaller numbers !!! Note Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface later. + +## Overriding pre-defined spaces + +To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_space`, `trailing_space`), define a nested class called Hyperopt and define the required spaces as follows: + +```python +class MyAwesomeStrategy(IStrategy): + class HyperOpt: + # Define a custom stoploss space. + def stoploss_space(self): + return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')] +``` + +## Space options + +For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types: + +* `Categorical` - Pick from a list of categories (e.g. `Categorical(['a', 'b', 'c'], name="cat")`) +* `Integer` - Pick from a range of whole numbers (e.g. `Integer(1, 10, name='rsi')`) +* `SKDecimal` - Pick from a range of decimal numbers with limited precision (e.g. `SKDecimal(0.1, 0.5, decimals=3, name='adx')`). *Available only with freqtrade*. +* `Real` - Pick from a range of decimal numbers with full precision (e.g. `Real(0.1, 0.5, name='adx')` + +You can import all of these from `freqtrade.optimize.space`, although `Categorical`, `Integer` and `Real` are only aliases for their corresponding scikit-optimize Spaces. `SKDecimal` is provided by freqtrade for faster optimizations. + +``` python +from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real # noqa +``` + +!!! Hint "SKDecimal vs. Real" + We recommend to use `SKDecimal` instead of the `Real` space in almost all cases. While the Real space provides full accuracy (up to ~16 decimal places) - this precision is rarely needed, and leads to unnecessary long hyperopt times. + + Assuming the definition of a rather small space (`SKDecimal(0.10, 0.15, decimals=2, name='xxx')`) - SKDecimal will have 5 possibilities (`[0.10, 0.11, 0.12, 0.13, 0.14, 0.15]`). + + A corresponding real space `Real(0.10, 0.15 name='xxx')` on the other hand has an almost unlimited number of possibilities (`[0.10, 0.010000000001, 0.010000000002, ... 0.014999999999, 0.01500000000]`). + +--- + +## Legacy Hyperopt + +This Section explains the configuration of an explicit Hyperopt file (separate to the strategy). + +!!! Warning "Deprecated / legacy mode" + Since the 2021.4 release you no longer have to write a separate hyperopt class, but all strategies can be hyperopted. + Please read the [main hyperopt page](hyperopt.md) for more details. + +### Prepare hyperopt file + +Configuring an explicit hyperopt file is similar to writing your own strategy, and many tasks will be similar. + +!!! Tip "About this page" + For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class. + +#### Create a Custom Hyperopt File + +The simplest way to get started is to use the following command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`. + +Let assume you want a hyperopt file `AwesomeHyperopt.py`: + +``` bash +freqtrade new-hyperopt --hyperopt AwesomeHyperopt +``` + +#### Legacy Hyperopt checklist + +Checklist on all tasks / possibilities in hyperopt + +Depending on the space you want to optimize, only some of the below are required: + +* fill `buy_strategy_generator` - for buy signal optimization +* fill `indicator_space` - for buy signal optimization +* fill `sell_strategy_generator` - for sell signal optimization +* fill `sell_indicator_space` - for sell signal optimization + +!!! Note + `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. + +Optional in hyperopt - can also be loaded from a strategy (recommended): + +* `populate_indicators` - fallback to create indicators +* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy +* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy + +!!! Note + You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods. + Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. + +Rarely you may also need to override: + +* `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default) +* `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps) +* `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default) +* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default) + +#### Defining a buy signal optimization + +Let's say you are curious: should you use MACD crossings or lower Bollinger +Bands to trigger your buys. And you also wonder should you use RSI or ADX to +help with those buy decisions. If you decide to use RSI or ADX, which values +should I use for them? So let's use hyperparameter optimization to solve this +mystery. + +We will start by defining a search space: + +```python + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching strategy parameters + """ + return [ + Integer(20, 40, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal'], name='trigger') + ] +``` + +Above definition says: I have five parameters I want you to randomly combine +to find the best combination. Two of them are integer values (`adx-value` and `rsi-value`) and I want you test in the range of values 20 to 40. +Then we have three category variables. First two are either `True` or `False`. +We use these to either enable or disable the ADX and RSI guards. +The last one we call `trigger` and use it to decide which buy trigger we want to use. + +So let's write the buy strategy generator using these values: + +```python + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by Hyperopt. + """ + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if 'adx-enabled' in params and params['adx-enabled']: + conditions.append(dataframe['adx'] > params['adx-value']) + if 'rsi-enabled' in params and params['rsi-enabled']: + conditions.append(dataframe['rsi'] < params['rsi-value']) + + # TRIGGERS + if 'trigger' in params: + if params['trigger'] == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if params['trigger'] == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) + + # Check that volume is not 0 + conditions.append(dataframe['volume'] > 0) + + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + + return dataframe + + return populate_buy_trend +``` + +Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations. +It will use the given historical data and make buys based on the buy signals generated with the above function. +Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)). + +!!! Note + The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. + When you want to test an indicator that isn't used by the bot currently, remember to + add it to the `populate_indicators()` method in your strategy or hyperopt file. + +#### Sell optimization + +Similar to the buy-signal above, sell-signals can also be optimized. +Place the corresponding settings into the following methods + +* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. +* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters. + +The configuration and rules are the same than for buy signals. +To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. + +### Execute Hyperopt + +Once you have updated your hyperopt configuration you can run it. +Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. More time usually results in better results. + +We strongly recommend to use `screen` or `tmux` to prevent any connection loss. + +```bash +freqtrade hyperopt --config config.json --hyperopt --hyperopt-loss --strategy -e 500 --spaces all +``` + +Use `` as the name of the custom hyperopt used. + +The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs. +Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results. + +The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below. + +!!! Note + Hyperopt will store hyperopt results with the timestamp of the hyperopt start time. + Reading commands (`hyperopt-list`, `hyperopt-show`) can use `--hyperopt-filename ` to read and display older hyperopt results. + You can find a list of filenames with `ls -l user_data/hyperopt_results/`. + +#### Running Hyperopt using methods from a strategy + +Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided. + +```bash +freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy +``` + +### Understand the Hyperopt Result + +Once Hyperopt is completed you can use the result to create a new strategy. +Given the following result from hyperopt: + +``` +Best result: + + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 + +Buy hyperspace params: +{ 'adx-value': 44, + 'rsi-value': 29, + 'adx-enabled': False, + 'rsi-enabled': True, + 'trigger': 'bb_lower'} +``` + +You should understand this result like: + +* The buy trigger that worked best was `bb_lower`. +* You should not use ADX because `adx-enabled: False`) +* You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`) + +You have to look inside your strategy file into `buy_strategy_generator()` +method, what those values match to. + +So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block: + +```python +(dataframe['rsi'] < 29.0) +``` + +Translating your whole hyperopt result as the new buy-signal would then look like: + +```python +def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: + dataframe.loc[ + ( + (dataframe['rsi'] < 29.0) & # rsi-value + dataframe['close'] < dataframe['bb_lowerband'] # trigger + ), + 'buy'] = 1 + return dataframe +``` + +### Validate backtesting results + +Once the optimized parameters and conditions have been implemented into your strategy, you should backtest the strategy to make sure everything is working as expected. + +To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. + +Should results don't match, please double-check to make sure you transferred all conditions correctly. +Pay special care to the stoploss (and trailing stoploss) parameters, as these are often set in configuration files, which override changes to the strategy. +You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss` or `trailing_stop`). + +### Sharing methods with your strategy + +Hyperopt classes provide access to the Strategy via the `strategy` class attribute. +This can be a great way to reduce code duplication if used correctly, but will also complicate usage for inexperienced users. + +``` python +from pandas import DataFrame +from freqtrade.strategy.interface import IStrategy +import freqtrade.vendor.qtpylib.indicators as qtpylib + +class MyAwesomeStrategy(IStrategy): + + buy_params = { + 'rsi-value': 30, + 'adx-value': 35, + } + + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + return self.buy_strategy_generator(self.buy_params, dataframe, metadata) + + @staticmethod + def buy_strategy_generator(params, dataframe: DataFrame, metadata: dict) -> DataFrame: + dataframe.loc[ + ( + qtpylib.crossed_above(dataframe['rsi'], params['rsi-value']) & + dataframe['adx'] > params['adx-value']) & + dataframe['volume'] > 0 + ) + , 'buy'] = 1 + return dataframe + +class MyAwesomeHyperOpt(IHyperOpt): + ... + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by Hyperopt. + """ + def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: + # Call strategy's buy strategy generator + return self.StrategyClass.buy_strategy_generator(params, dataframe, metadata) + + return populate_buy_trend +``` diff --git a/docs/assets/ccxt-logo.svg b/docs/assets/ccxt-logo.svg new file mode 100644 index 000000000..e52682546 --- /dev/null +++ b/docs/assets/ccxt-logo.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/docs/assets/freqtrade_poweredby.svg b/docs/assets/freqtrade_poweredby.svg new file mode 100644 index 000000000..957ec6401 --- /dev/null +++ b/docs/assets/freqtrade_poweredby.svg @@ -0,0 +1,44 @@ + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + Freqtrade + + + + + poweredby + + diff --git a/docs/assets/telegram_forcebuy.png b/docs/assets/telegram_forcebuy.png new file mode 100644 index 000000000..b0592bff3 Binary files /dev/null and b/docs/assets/telegram_forcebuy.png differ diff --git a/docs/backtesting.md b/docs/backtesting.md index a14c8f2e4..2027c2079 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -15,15 +15,16 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] - [--eps] [--dmmp] [--enable-protections] + [-p PAIRS [PAIRS ...]] [--eps] [--dmmp] + [--enable-protections] + [--dry-run-wallet DRY_RUN_WALLET] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export EXPORT] [--export-filename PATH] optional arguments: -h, --help show this help message and exit -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME - Specify ticker interval (`1m`, `5m`, `30m`, `1h`, - `1d`). + Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`). --timerange TIMERANGE Specify what timerange of data to use. --data-format-ohlcv {json,jsongz,hdf5} @@ -37,6 +38,9 @@ optional arguments: setting. --fee FLOAT Specify fee ratio. Will be applied twice (on trade entry and exit). + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Limit command to these pairs. Pairs are space- + separated. --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). @@ -48,6 +52,9 @@ optional arguments: Enable protections for backtesting.Will slow backtesting down by a considerable amount, but will include configured protections + --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET + Starting balance, used for backtesting / hyperopt and + dry-runs. --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] Provide a space-separated list of strategies to backtest. Please note that ticker-interval needs to be @@ -91,8 +98,7 @@ Strategy arguments: ## Test your strategy with Backtesting Now you have good Buy and Sell strategies and some historic data, you want to test it against -real data. This is what we call -[backtesting](https://en.wikipedia.org/wiki/Backtesting). +real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHCLV) data from `user_data/data/` by default. If no data is available for the exchange / pair / timeframe combination, backtesting will ask you to download them first using `freqtrade download-data`. @@ -100,6 +106,8 @@ For details on downloading, please refer to the [Data Downloading](data-download The result of backtesting will confirm if your bot has better odds of making a profit than a loss. +All profit calculations include fees, and freqtrade will use the exchange's default fees for the calculation. + !!! Warning "Using dynamic pairlists for backtesting" Using dynamic pairlists is possible, however it relies on the current market conditions - which will not reflect the historic status of the pairlist. Also, when using pairlists other than StaticPairlist, reproducability of backtesting-results cannot be guaranteed. @@ -107,38 +115,56 @@ The result of backtesting will confirm if your bot has better odds of making a p To achieve reproducible results, best generate a pairlist via the [`test-pairlist`](utils.md#test-pairlist) command and use that as static pairlist. -### Run a backtesting against the currencies listed in your config file +### Starting balance -#### With 5 min candle (OHLCV) data (per default) +Backtesting will require a starting balance, which can be provided as `--dry-run-wallet ` or `--starting-balance ` command line argument, or via `dry_run_wallet` configuration setting. +This amount must be higher than `stake_amount`, otherwise the bot will not be able to simulate any trade. + +### Dynamic stake amount + +Backtesting supports [dynamic stake amount](configuration.md#dynamic-stake-amount) by configuring `stake_amount` as `"unlimited"`, which will split the starting balance into `max_open_trades` pieces. +Profits from early trades will result in subsequent higher stake amounts, resulting in compounding of profits over the backtesting period. + +### Example backtesting commands + +With 5 min candle (OHLCV) data (per default) ```bash -freqtrade backtesting +freqtrade backtesting --strategy AwesomeStrategy ``` -#### With 1 min candle (OHLCV) data +Where `--strategy AwesomeStrategy` / `-s AwesomeStrategy` refers to the class name of the strategy, which is within a python file in the `user_data/strategies` directory. + +--- + +With 1 min candle (OHLCV) data ```bash -freqtrade backtesting --timeframe 1m +freqtrade backtesting --strategy AwesomeStrategy --timeframe 1m ``` -#### Using a different on-disk historical candle (OHLCV) data source +--- + +Providing a custom starting balance of 1000 (in stake currency) + +```bash +freqtrade backtesting --strategy AwesomeStrategy --dry-run-wallet 1000 +``` + +--- + +Using a different on-disk historical candle (OHLCV) data source Assume you downloaded the history data from the Bittrex exchange and kept it in the `user_data/data/bittrex-20180101` directory. You can then use this data for backtesting as follows: ```bash -freqtrade --datadir user_data/data/bittrex-20180101 backtesting +freqtrade backtesting --strategy AwesomeStrategy --datadir user_data/data/bittrex-20180101 ``` -#### With a (custom) strategy file +--- -```bash -freqtrade backtesting -s SampleStrategy -``` - -Where `-s SampleStrategy` refers to the class name within the strategy file `sample_strategy.py` found in the `freqtrade/user_data/strategies` directory. - -#### Comparing multiple Strategies +Comparing multiple Strategies ```bash freqtrade backtesting --strategy-list SampleStrategy1 AwesomeStrategy --timeframe 5m @@ -146,23 +172,29 @@ freqtrade backtesting --strategy-list SampleStrategy1 AwesomeStrategy --timefram Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies. -#### Exporting trades to file +--- + +Exporting trades to file ```bash -freqtrade backtesting --export trades --config config.json --strategy SampleStrategy +freqtrade backtesting --strategy backtesting --export trades --config config.json ``` The exported trades can be used for [further analysis](#further-backtest-result-analysis), or can be used by the plotting script `plot_dataframe.py` in the scripts directory. -#### Exporting trades to file specifying a custom filename +--- + +Exporting trades to file specifying a custom filename ```bash -freqtrade backtesting --export trades --export-filename=backtest_samplestrategy.json +freqtrade backtesting --strategy backtesting --export trades --export-filename=backtest_samplestrategy.json ``` Please also read about the [strategy startup period](strategy-customization.md#strategy-startup-period). -#### Supplying custom fee value +--- + +Supplying custom fee value Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt. To account for this in backtesting, you can use the `--fee` command line option to supply this value to backtesting. @@ -177,26 +209,26 @@ freqtrade backtesting --fee 0.001 !!! Note Only supply this option (or the corresponding configuration parameter) if you want to experiment with different fee values. By default, Backtesting fetches the default fee from the exchange pair/market info. -#### Running backtest with smaller testset by using timerange +--- -Use the `--timerange` argument to change how much of the testset you want to use. +Running backtest with smaller test-set by using timerange +Use the `--timerange` argument to change how much of the test-set you want to use. -For example, running backtesting with the `--timerange=20190501-` option will use all available data starting with May 1st, 2019 from your inputdata. +For example, running backtesting with the `--timerange=20190501-` option will use all available data starting with May 1st, 2019 from your input data. ```bash freqtrade backtesting --timerange=20190501- ``` -You can also specify particular dates or a range span indexed by start and stop. +You can also specify particular date ranges. The full timerange specification: -- Use tickframes till 2018/01/31: `--timerange=-20180131` -- Use tickframes since 2018/01/31: `--timerange=20180131-` -- Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` -- Use tickframes between POSIX timestamps 1527595200 1527618600: - `--timerange=1527595200-1527618600` +- Use data until 2018/01/31: `--timerange=-20180131` +- Use data since 2018/01/31: `--timerange=20180131-` +- Use data since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` +- Use data between POSIX / epoch timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` ## Understand the backtesting result @@ -205,29 +237,29 @@ The most important in the backtesting is to understand the result. A backtesting result will look like that: ``` -========================================================= BACKTESTING REPORT ======================================================== -| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | -|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:| -| ADA/BTC | 35 | -0.11 | -3.88 | -0.00019428 | -1.94 | 4:35:00 | 14 | 0 | 21 | -| ARK/BTC | 11 | -0.41 | -4.52 | -0.00022647 | -2.26 | 2:03:00 | 3 | 0 | 8 | -| BTS/BTC | 32 | 0.31 | 9.78 | 0.00048938 | 4.89 | 5:05:00 | 18 | 0 | 14 | -| DASH/BTC | 13 | -0.08 | -1.07 | -0.00005343 | -0.53 | 4:39:00 | 6 | 0 | 7 | -| ENG/BTC | 18 | 1.36 | 24.54 | 0.00122807 | 12.27 | 2:50:00 | 8 | 0 | 10 | -| EOS/BTC | 36 | 0.08 | 3.06 | 0.00015304 | 1.53 | 3:34:00 | 16 | 0 | 20 | -| ETC/BTC | 26 | 0.37 | 9.51 | 0.00047576 | 4.75 | 6:14:00 | 11 | 0 | 15 | -| ETH/BTC | 33 | 0.30 | 9.96 | 0.00049856 | 4.98 | 7:31:00 | 16 | 0 | 17 | -| IOTA/BTC | 32 | 0.03 | 1.09 | 0.00005444 | 0.54 | 3:12:00 | 14 | 0 | 18 | -| LSK/BTC | 15 | 1.75 | 26.26 | 0.00131413 | 13.13 | 2:58:00 | 6 | 0 | 9 | -| LTC/BTC | 32 | -0.04 | -1.38 | -0.00006886 | -0.69 | 4:49:00 | 11 | 0 | 21 | -| NANO/BTC | 17 | 1.26 | 21.39 | 0.00107058 | 10.70 | 1:55:00 | 10 | 0 | 7 | -| NEO/BTC | 23 | 0.82 | 18.97 | 0.00094936 | 9.48 | 2:59:00 | 10 | 0 | 13 | -| REQ/BTC | 9 | 1.17 | 10.54 | 0.00052734 | 5.27 | 3:47:00 | 4 | 0 | 5 | -| XLM/BTC | 16 | 1.22 | 19.54 | 0.00097800 | 9.77 | 3:15:00 | 7 | 0 | 9 | -| XMR/BTC | 23 | -0.18 | -4.13 | -0.00020696 | -2.07 | 5:30:00 | 12 | 0 | 11 | -| XRP/BTC | 35 | 0.66 | 22.96 | 0.00114897 | 11.48 | 3:49:00 | 12 | 0 | 23 | -| ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 | 0 | 15 | -| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 | -========================================================= SELL REASON STATS ========================================================= +========================================================= BACKTESTING REPORT ========================================================== +| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins Draws Loss Win% | +|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:-------------|-------------------------:| +| ADA/BTC | 35 | -0.11 | -3.88 | -0.00019428 | -1.94 | 4:35:00 | 14 0 21 40.0 | +| ARK/BTC | 11 | -0.41 | -4.52 | -0.00022647 | -2.26 | 2:03:00 | 3 0 8 27.3 | +| BTS/BTC | 32 | 0.31 | 9.78 | 0.00048938 | 4.89 | 5:05:00 | 18 0 14 56.2 | +| DASH/BTC | 13 | -0.08 | -1.07 | -0.00005343 | -0.53 | 4:39:00 | 6 0 7 46.2 | +| ENG/BTC | 18 | 1.36 | 24.54 | 0.00122807 | 12.27 | 2:50:00 | 8 0 10 44.4 | +| EOS/BTC | 36 | 0.08 | 3.06 | 0.00015304 | 1.53 | 3:34:00 | 16 0 20 44.4 | +| ETC/BTC | 26 | 0.37 | 9.51 | 0.00047576 | 4.75 | 6:14:00 | 11 0 15 42.3 | +| ETH/BTC | 33 | 0.30 | 9.96 | 0.00049856 | 4.98 | 7:31:00 | 16 0 17 48.5 | +| IOTA/BTC | 32 | 0.03 | 1.09 | 0.00005444 | 0.54 | 3:12:00 | 14 0 18 43.8 | +| LSK/BTC | 15 | 1.75 | 26.26 | 0.00131413 | 13.13 | 2:58:00 | 6 0 9 40.0 | +| LTC/BTC | 32 | -0.04 | -1.38 | -0.00006886 | -0.69 | 4:49:00 | 11 0 21 34.4 | +| NANO/BTC | 17 | 1.26 | 21.39 | 0.00107058 | 10.70 | 1:55:00 | 10 0 7 58.5 | +| NEO/BTC | 23 | 0.82 | 18.97 | 0.00094936 | 9.48 | 2:59:00 | 10 0 13 43.5 | +| REQ/BTC | 9 | 1.17 | 10.54 | 0.00052734 | 5.27 | 3:47:00 | 4 0 5 44.4 | +| XLM/BTC | 16 | 1.22 | 19.54 | 0.00097800 | 9.77 | 3:15:00 | 7 0 9 43.8 | +| XMR/BTC | 23 | -0.18 | -4.13 | -0.00020696 | -2.07 | 5:30:00 | 12 0 11 52.2 | +| XRP/BTC | 35 | 0.66 | 22.96 | 0.00114897 | 11.48 | 3:49:00 | 12 0 23 34.3 | +| ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 0 15 31.8 | +| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 0 243 43.4 | +========================================================= SELL REASON STATS ========================================================== | Sell Reason | Sells | Wins | Draws | Losses | |:-------------------|--------:|------:|-------:|--------:| | trailing_stop_loss | 205 | 150 | 0 | 55 | @@ -235,11 +267,11 @@ A backtesting result will look like that: | sell_signal | 56 | 36 | 0 | 20 | | force_sell | 2 | 0 | 0 | 2 | ====================================================== LEFT OPEN TRADES REPORT ====================================================== -| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | -|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:| -| ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 | 0 | 0 | -| LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 | 0 | 0 | -| TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 | 0 | 0 | +| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Win Draw Loss Win% | +|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|--------------------:| +| ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 0 0 100 | +| LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 0 0 100 | +| TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 0 0 100 | =============== SUMMARY METRICS =============== | Metric | Value | |-----------------------+---------------------| @@ -248,19 +280,32 @@ A backtesting result will look like that: | Max open trades | 3 | | | | | Total trades | 429 | -| Total Profit % | 152.41% | +| Starting balance | 0.01000000 BTC | +| Final balance | 0.01762792 BTC | +| Absolute profit | 0.00762792 BTC | +| Total profit % | 76.2% | | Trades per day | 3.575 | +| Avg. stake amount | 0.001 BTC | +| Total trade volume | 0.429 BTC | | | | | Best Pair | LSK/BTC 26.26% | | Worst Pair | ZEC/BTC -10.18% | | Best Trade | LSK/BTC 4.25% | | Worst Trade | ZEC/BTC -10.25% | -| Best day | 25.27% | -| Worst day | -30.67% | +| Best day | 0.00076 BTC | +| Worst day | -0.00036 BTC | +| Days win/draw/lose | 12 / 82 / 25 | | Avg. Duration Winners | 4:23:00 | | Avg. Duration Loser | 6:55:00 | +| Zero Duration Trades | 4.6% (20) | +| Rejected Buy signals | 3089 | | | | -| Max Drawdown | 50.63% | +| Min balance | 0.00945123 BTC | +| Max balance | 0.01846651 BTC | +| Drawdown | 50.63% | +| Drawdown | 0.0015 BTC | +| Drawdown high | 0.0013 BTC | +| Drawdown low | -0.0002 BTC | | Drawdown Start | 2019-02-15 14:10:00 | | Drawdown End | 2019-04-11 18:15:00 | | Market change | -5.88% | @@ -275,15 +320,15 @@ The last line will give you the overall performance of your strategy, here: ``` -| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 243 | +| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 0 243 43.4 | ``` The bot has made `429` trades for an average duration of `4:12:00`, with a performance of `76.20%` (profit), that means it has earned a total of `0.00762792 BTC` starting with a capital of 0.01 BTC. -The column `avg profit %` shows the average profit for all trades made while the column `cum profit %` sums up all the profits/losses. -The column `tot profit %` shows instead the total profit % in relation to allocated capital (`max_open_trades * stake_amount`). -In the above results we have `max_open_trades=2` and `stake_amount=0.005` in config so `tot_profit %` will be `(76.20/100) * (0.005 * 2) =~ 0.00762792 BTC`. +The column `Avg Profit %` shows the average profit for all trades made while the column `Cum Profit %` sums up all the profits/losses. +The column `Tot Profit %` shows instead the total profit % in relation to the starting balance. +In the above results, we have a starting balance of 0.01 BTC and an absolute profit of 0.00762792 BTC - so the `Tot Profit %` will be `(0.00762792 / 0.01) * 100 ~= 76.2%`. Your strategy performance is influenced by your buy strategy, your sell strategy, and also by the `minimal_roi` and `stop_loss` you have set. @@ -324,19 +369,32 @@ It contains some useful key metrics about performance of your strategy on backte | Max open trades | 3 | | | | | Total trades | 429 | -| Total Profit % | 152.41% | +| Starting balance | 0.01000000 BTC | +| Final balance | 0.01762792 BTC | +| Absolute profit | 0.00762792 BTC | +| Total profit % | 76.2% | | Trades per day | 3.575 | +| Avg. stake amount | 0.001 BTC | +| Total trade volume | 0.429 BTC | | | | | Best Pair | LSK/BTC 26.26% | | Worst Pair | ZEC/BTC -10.18% | | Best Trade | LSK/BTC 4.25% | | Worst Trade | ZEC/BTC -10.25% | -| Best day | 25.27% | -| Worst day | -30.67% | +| Best day | 0.00076 BTC | +| Worst day | -0.00036 BTC | +| Days win/draw/lose | 12 / 82 / 25 | | Avg. Duration Winners | 4:23:00 | | Avg. Duration Loser | 6:55:00 | +| Zero Duration Trades | 4.6% (20) | +| Rejected Buy signals | 3089 | | | | -| Max Drawdown | 50.63% | +| Min balance | 0.00945123 BTC | +| Max balance | 0.01846651 BTC | +| Drawdown | 50.63% | +| Drawdown | 0.0015 BTC | +| Drawdown high | 0.0013 BTC | +| Drawdown low | -0.0002 BTC | | Drawdown Start | 2019-02-15 14:10:00 | | Drawdown End | 2019-04-11 18:15:00 | | Market change | -5.88% | @@ -347,13 +405,23 @@ It contains some useful key metrics about performance of your strategy on backte - `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option). - `Max open trades`: Setting of `max_open_trades` (or `--max-open-trades`) - or number of pairs in the pairlist (whatever is lower). - `Total trades`: Identical to the total trades of the backtest output table. -- `Total Profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. +- `Starting balance`: Start balance - as given by dry-run-wallet (config or command line). +- `Final balance`: Final balance - starting balance + absolute profit. +- `Absolute profit`: Profit made in stake currency. +- `Total profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. Calculated as `(End capital − Starting capital) / Starting capital`. - `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). +- `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount. +- `Total trade volume`: Volume generated on the exchange to reach the above profit. - `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. -- `Best Trade` / `Worst Trade`: Biggest winning trade and biggest losing trade +- `Best Trade` / `Worst Trade`: Biggest single winning trade and biggest single losing trade. - `Best day` / `Worst day`: Best and worst day based on daily profit. +- `Days win/draw/lose`: Winning / Losing days (draws are usually days without closed trade). - `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades. -- `Max Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced). +- `Zero Duration Trades`: A number of trades that completed within same candle as they opened and had `trailing_stop_loss` sell reason. A significant amount of such trades may indicate that strategy is exploiting trailing stoploss behavior in backtesting and produces unrealistic results. +- `Rejected Buy signals`: Buy signals that could not be acted upon due to max_open_trades being reached. +- `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period. +- `Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced). +- `Drawdown high` / `Drawdown low`: Profit at the beginning and end of the largest drawdown period. A negative low value means initial capital lost. - `Drawdown Start` / `Drawdown End`: Start and end datetime for this largest drawdown (can also be visualized via the `plot-dataframe` sub-command). - `Market change`: Change of the market during the backtest period. Calculated as average of all pairs changes from the first to the last candle using the "close" column. @@ -362,6 +430,7 @@ It contains some useful key metrics about performance of your strategy on backte Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: - Buys happen at open-price +- All orders are filled at the requested price (no slippage, no unfilled orders) - Sell-signal sells happen at open-price of the consecutive candle - Sell-signal is favored over Stoploss, because sell-signals are assumed to trigger on candle's open - ROI @@ -409,15 +478,14 @@ There will be an additional table comparing win/losses of the different strategi Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy. ``` -=========================================================== STRATEGY SUMMARY =========================================================== -| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | -|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:| -| Strategy1 | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 | -| Strategy2 | 1487 | -0.13 | -197.58 | -0.00988917 | -98.79 | 4:43:00 | 662 | 0 | 825 | +=========================================================== STRATEGY SUMMARY ========================================================================= +| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | Drawdown % | +|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|-----------:| +| Strategy1 | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 | 45.2 | +| Strategy2 | 1487 | -0.13 | -197.58 | -0.00988917 | -98.79 | 4:43:00 | 662 | 0 | 825 | 241.68 | ``` ## Next step -Great, your strategy is profitable. What if the bot can give your the -optimal parameters to use for your strategy? +Great, your strategy is profitable. What if the bot can give your the optimal parameters to use for your strategy? Your next step is to learn [how to find optimal parameters with Hyperopt](hyperopt.md) diff --git a/docs/bot-basics.md b/docs/bot-basics.md index 30a25d4fc..943af0362 100644 --- a/docs/bot-basics.md +++ b/docs/bot-basics.md @@ -4,14 +4,14 @@ This page provides you some basic concepts on how Freqtrade works and operates. ## Freqtrade terminology -* Strategy: Your trading strategy, telling the bot what to do. -* Trade: Open position. -* Open Order: Order which is currently placed on the exchange, and is not yet complete. -* Pair: Tradable pair, usually in the format of Quote/Base (e.g. XRP/USDT). -* Timeframe: Candle length to use (e.g. `"5m"`, `"1h"`, ...). -* Indicators: Technical indicators (SMA, EMA, RSI, ...). -* Limit order: Limit orders which execute at the defined limit price or better. -* Market order: Guaranteed to fill, may move price depending on the order size. +* **Strategy**: Your trading strategy, telling the bot what to do. +* **Trade**: Open position. +* **Open Order**: Order which is currently placed on the exchange, and is not yet complete. +* **Pair**: Tradable pair, usually in the format of Quote/Base (e.g. XRP/USDT). +* **Timeframe**: Candle length to use (e.g. `"5m"`, `"1h"`, ...). +* **Indicators**: Technical indicators (SMA, EMA, RSI, ...). +* **Limit order**: Limit orders which execute at the defined limit price or better. +* **Market order**: Guaranteed to fill, may move price depending on the order size. ## Fee handling @@ -53,6 +53,7 @@ This loop will be repeated again and again until the bot is stopped. * Calls `bot_loop_start()` once. * Calculate indicators (calls `populate_indicators()` once per pair). * Calculate buy / sell signals (calls `populate_buy_trend()` and `populate_sell_trend()` once per pair) +* Confirm trade buy / sell (calls `confirm_trade_entry()` and `confirm_trade_exit()` if implemented in the strategy) * Loops per candle simulating entry and exit points. * Generate backtest report output diff --git a/docs/bot-usage.md b/docs/bot-usage.md index c7fe8634d..b65220722 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -56,6 +56,7 @@ optional arguments: usage: freqtrade trade [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [--db-url PATH] [--sd-notify] [--dry-run] + [--dry-run-wallet DRY_RUN_WALLET] optional arguments: -h, --help show this help message and exit @@ -66,6 +67,9 @@ optional arguments: --sd-notify Notify systemd service manager. --dry-run Enforce dry-run for trading (removes Exchange secrets and simulates trades). + --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET + Starting balance, used for backtesting / hyperopt and + dry-runs. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/configuration.md b/docs/configuration.md index 00d2830e4..ef6f34094 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,7 +11,16 @@ Per default, the bot loads the configuration from the `config.json` file, locate You can specify a different configuration file used by the bot with the `-c/--config` command line option. -In some advanced use cases, multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream. +Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream. + +!!! Tip "Use multiple configuration files to keep secrets secret" + You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself. + + ``` bash + freqtrade trade --config user_data/config.json --config user_data/config-private.json <...> + ``` + The 2nd file should only specify what you intend to override. + If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`). If you used the [Quick start](installation.md/#quick-start) method for installing the bot, the installation script should have already created the default configuration file (`config.json`) for you. @@ -40,8 +49,8 @@ Mandatory parameters are marked as **Required**, which means that they are requi | Parameter | Description | |------------|-------------| | `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the length of your pairlist is another limitation which can apply. If -1 then it is ignored (i.e. potentially unlimited open trades, limited by the pairlist). [More information below](#configuring-amount-per-trade).
**Datatype:** Positive integer or -1. -| `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String -| `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Positive float or `"unlimited"`. +| `stake_currency` | **Required.** Crypto-currency used for trading.
**Datatype:** String +| `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade).
**Datatype:** Positive float or `"unlimited"`. | `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade).
*Defaults to `0.99` 99%).*
**Datatype:** Positive float between `0.1` and `1.0`. | `amend_last_stake_amount` | Use reduced last stake amount if necessary. [More information below](#configuring-amount-per-trade).
*Defaults to `false`.*
**Datatype:** Boolean | `last_stake_amount_min_ratio` | Defines minimum stake amount that has to be left and executed. Applies only to the last stake amount when it's amended to a reduced value (i.e. if `amend_last_stake_amount` is set to `true`). [More information below](#configuring-amount-per-trade).
*Defaults to `0.5`.*
**Datatype:** Float (as ratio) @@ -49,7 +58,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `timeframe` | The timeframe (former ticker interval) to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String | `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
**Datatype:** String | `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
**Datatype:** Boolean -| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in the Dry Run mode.
*Defaults to `1000`.*
**Datatype:** Float +| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in Dry Run mode.
*Defaults to `1000`.*
**Datatype:** Float | `cancel_open_orders_on_exit` | Cancel open orders when the `/stop` RPC command is issued, `Ctrl+C` is pressed or the bot dies unexpectedly. When set to `true`, this allows you to use `/stop` to cancel unfilled and partially filled orders in the event of a market crash. It does not impact open positions.
*Defaults to `false`.*
**Datatype:** Boolean | `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `minimal_roi` | **Required.** Set the threshold as ratio the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict @@ -58,15 +67,18 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-custom-positive-loss). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Float | `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
**Datatype:** Float | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean -| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer -| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer +| `fee` | Fee used during backtesting / dry-runs. Should normally not be configured, which has freqtrade fall back to the exchange default fee. Set as ratio (e.g. 0.001 = 0.1%). Fee is applied twice for each trade, once when buying, once when selling.
**Datatype:** Float (as ratio) +| `unfilledtimeout.buy` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer +| `unfilledtimeout.sell` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer +| `unfilledtimeout.unit` | Unit to use in unfilledtimeout setting. Note: If you set unfilledtimeout.unit to "seconds", "internals.process_throttle_secs" must be inferior or equal to timeout [Strategy Override](#parameters-in-the-strategy).
*Defaults to `minutes`.*
**Datatype:** String | `bid_strategy.price_side` | Select the side of the spread the bot should look at to get the buy rate. [More information below](#buy-price-side).
*Defaults to `bid`.*
**Datatype:** String (either `ask` or `bid`). -| `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#buy-price-without-orderbook-enabled). +| `bid_strategy.ask_last_balance` | **Required.** Interpolate the bidding price. More information [below](#buy-price-without-orderbook-enabled). | `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled).
**Datatype:** Boolean | `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids to buy. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in [Order Book Bids](#buy-price-with-orderbook-enabled).
*Defaults to `1`.*
**Datatype:** Positive Integer | `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book. [Check market depth](#check-depth-of-market).
*Defaults to `false`.*
**Datatype:** Boolean | `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The difference ratio of buy orders and sell orders found in Order Book. A value below 1 means sell order size is greater, while value greater than 1 means buy order size is higher. [Check market depth](#check-depth-of-market)
*Defaults to `0`.*
**Datatype:** Float (as ratio) | `ask_strategy.price_side` | Select the side of the spread the bot should look at to get the sell rate. [More information below](#sell-price-side).
*Defaults to `ask`.*
**Datatype:** String (either `ask` or `bid`). +| `ask_strategy.bid_last_balance` | Interpolate the selling price. More information [below](#sell-price-without-orderbook-enabled). | `ask_strategy.use_order_book` | Enable selling of open trades using [Order Book Asks](#sell-price-with-orderbook-enabled).
**Datatype:** Boolean | `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer | `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer @@ -97,6 +109,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `telegram.balance_dust_level` | Dust-level (in stake currency) - currencies with a balance below this will not be shown by `/balance`.
**Datatype:** float | `webhook.enabled` | Enable usage of Webhook notifications
**Datatype:** Boolean | `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String | `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String @@ -141,8 +154,6 @@ Values set in the configuration file always overwrite values set in the strategy * `process_only_new_candles` * `order_types` * `order_time_in_force` -* `stake_currency` -* `stake_amount` * `unfilledtimeout` * `disable_dataframe_checks` * `protections` @@ -156,6 +167,23 @@ Values set in the configuration file always overwrite values set in the strategy There are several methods to configure how much of the stake currency the bot will use to enter a trade. All methods respect the [available balance configuration](#available-balance) as explained below. +#### Minimum trade stake + +The minimum stake amount will depend by exchange and pair, and is usually listed in the exchange support pages. +Assuming the minimum tradable amount for XRP/USD is 20 XRP (given by the exchange), and the price is 0.4$. + +The minimum stake amount to buy this pair is therefore `20 * 0.6 ~= 12`. +This exchange has also a limit on USD - where all orders must be > 10$ - which however does not apply in this case. + +To guarantee safe execution, freqtrade will not allow buying with a stake-amount of 10.1$, instead, it'll make sure that there's enough space to place a stoploss below the pair (+ an offset, defined by `amount_reserve_percent`, which defaults to 5%). + +With a reserve of 5%, the minimum stake amount would be ~12.6$ (`12 * (1 + 0.05)`). If we take in account a stoploss of 10% on top of that - we'd end up with a value of ~14$ (`12.6 / (1 - 0.1)`). + +To limit this calculation in case of large stoploss values, the calculated minimum stake-limit will never be more than 50% above the real limit. + +!!! Warning + Since the limits on exchanges are usually stable and are not updated often, some pairs can show pretty high minimum limits, simply because the price increased a lot since the last limit adjustment by the exchange. + #### Available balance By default, the bot assumes that the `complete amount - 1%` is at it's disposal, and when using [dynamic stake amount](#dynamic-stake-amount), it will split the complete balance into `max_open_trades` buckets per trade. @@ -218,11 +246,14 @@ To allow the bot to trade all the available `stake_currency` in your account (mi "tradable_balance_ratio": 0.99, ``` -!!! Note - This configuration will allow increasing / decreasing stakes depending on the performance of the bot (lower stake if bot is loosing, higher stakes if the bot has a winning record, since higher balances are available). +!!! Tip "Compounding profits" + This configuration will allow increasing / decreasing stakes depending on the performance of the bot (lower stake if bot is loosing, higher stakes if the bot has a winning record, since higher balances are available), and will result in profit compounding. !!! Note "When using Dry-Run Mode" - When using `"stake_amount" : "unlimited",` in combination with Dry-Run, the balance will be simulated starting with a stake of `dry_run_wallet` which will evolve over time. It is therefore important to set `dry_run_wallet` to a sensible value (like 0.05 or 0.01 for BTC and 1000 or 100 for USDT, for example), otherwise it may simulate trades with 100 BTC (or more) or 0.05 USDT (or less) at once - which may not correspond to your real available balance or is less than the exchange minimal limit for the order amount for the stake currency. + When using `"stake_amount" : "unlimited",` in combination with Dry-Run, Backtesting or Hyperopt, the balance will be simulated starting with a stake of `dry_run_wallet` which will evolve over time. + It is therefore important to set `dry_run_wallet` to a sensible value (like 0.05 or 0.01 for BTC and 1000 or 100 for USDT, for example), otherwise it may simulate trades with 100 BTC (or more) or 0.05 USDT (or less) at once - which may not correspond to your real available balance or is less than the exchange minimal limit for the order amount for the stake currency. + +--8<-- "includes/pricing.md" ### Understand minimal_roi @@ -273,9 +304,12 @@ For example, if your strategy is using a 1h timeframe, and you only want to buy }, ``` +!!! Note + This setting resets with each new candle, so it will not prevent sticking-signals from executing on the 2nd or 3rd candle they're active. Best use a "trigger" selector for buy signals, which are only active for one candle. + ### Understand order_types -The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`, `emergencysell`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. +The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`, `emergencysell`, `forcesell`, `forcebuy`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. This allows to buy using limit orders, sell using limit-orders, and create stoplosses using market orders. It also allows to set the @@ -287,7 +321,7 @@ the buy order is fulfilled. If this is configured, the following 4 values (`buy`, `sell`, `stoploss` and `stoploss_on_exchange`) need to be present, otherwise the bot will fail to start. -For information on (`emergencysell`,`stoploss_on_exchange`,`stoploss_on_exchange_interval`,`stoploss_on_exchange_limit_ratio`) please see stop loss documentation [stop loss on exchange](stoploss.md) +For information on (`emergencysell`,`forcesell`, `forcebuy`, `stoploss_on_exchange`,`stoploss_on_exchange_interval`,`stoploss_on_exchange_limit_ratio`) please see stop loss documentation [stop loss on exchange](stoploss.md) Syntax for Strategy: @@ -296,6 +330,8 @@ order_types = { "buy": "limit", "sell": "limit", "emergencysell": "market", + "forcebuy": "market", + "forcesell": "market", "stoploss": "market", "stoploss_on_exchange": False, "stoploss_on_exchange_interval": 60, @@ -310,6 +346,8 @@ Configuration: "buy": "limit", "sell": "limit", "emergencysell": "market", + "forcebuy": "market", + "forcesell": "market", "stoploss": "market", "stoploss_on_exchange": false, "stoploss_on_exchange_interval": 60 @@ -368,8 +406,8 @@ The possible values are: `gtc` (default), `fok` or `ioc`. ``` !!! Warning - This is an ongoing work. For now it is supported only for binance and only for buy orders. - Please don't change the default value unless you know what you are doing. + This is an ongoing work. For now it is supported only for binance. + Please don't change the default value unless you know what you are doing and have researched the impact of using different values. ### Exchange configuration @@ -410,26 +448,6 @@ This configuration enables binance, as well as rate limiting to avoid bans from Optimal settings for rate limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. -#### Advanced Freqtrade Exchange configuration - -Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behaviours. - -Available options are listed in the exchange-class as `_ft_has_default`. - -For example, to test the order type `FOK` with Kraken, and modify candle limit to 200 (so you only get 200 candles per API call): - -```json -"exchange": { - "name": "kraken", - "_ft_has_params": { - "order_time_in_force": ["gtc", "fok"], - "ohlcv_candle_limit": 200 - } -``` - -!!! Warning - Please make sure to fully understand the impacts of these settings before modifying them. - ### What values can be used for fiat_display_currency? The `fiat_display_currency` configuration parameter sets the base currency to use for the @@ -449,8 +467,6 @@ The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" ``` ---8<-- "includes/pricing.md" - ## Using Dry-run mode We recommend starting the bot in the Dry-run mode to see how your bot will @@ -515,16 +531,27 @@ API Keys are usually only required for live trading (trading for real money, bot **Insert your Exchange API key (change them by fake api keys):** ```json -"exchange": { +{ + "exchange": { "name": "bittrex", "key": "af8ddd35195e9dc500b9a6f799f6f5c93d89193b", "secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5", - ... + //"password": "", // Optional, not needed by all exchanges) + // ... + } + //... } ``` You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange. +!!! Hint "Keep your secrets secret" + To keep your secrets secret, we recommend to use a 2nd configuration for your API keys. + Simply use the above snippet in a new configuration file (e.g. `config-private.json`) and keep your settings in this file. + You can then start the bot with `freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>` to have your keys loaded. + + **NEVER** share your private configuration file or your exchange keys with anyone! + ### Using proxy with Freqtrade To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. diff --git a/docs/data-download.md b/docs/data-download.md index 04f444a8b..01561c89b 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -11,8 +11,9 @@ Otherwise `--exchange` becomes mandatory. You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used. !!! Tip "Tip: Updating existing data" - If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. - Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. + If you already have backtesting data available in your data-directory and would like to refresh this data up to today, do not use `--days` or `--timerange` parameters. Freqtrade will keep the available data and only download the missing data. + If you are updating existing data after inserting new pairs that you have no data for, use `--new-pairs-days xx` parameter. Specified number of days will be downloaded for new pairs while old pairs will be updated with missing data only. + If you use `--days xx` parameter alone - data for specified number of days will be downloaded for _all_ pairs. Be careful, if specified number of days is smaller than gap between now and last downloaded candle - freqtrade will delete all existing data to avoid gaps in candle data. ### Usage @@ -20,8 +21,9 @@ You can use a relative timerange (`--days 20`) or an absolute starting point (`- usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] [--pairs-file FILE] - [--days INT] [--timerange TIMERANGE] - [--dl-trades] [--exchange EXCHANGE] + [--days INT] [--new-pairs-days INT] + [--timerange TIMERANGE] [--dl-trades] + [--exchange EXCHANGE] [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] [--erase] [--data-format-ohlcv {json,jsongz,hdf5}] @@ -30,10 +32,12 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] - Show profits for only these pairs. Pairs are space- + Limit command to these pairs. Pairs are space- separated. --pairs-file FILE File containing a list of pairs to download. --days INT Download data for given number of days. + --new-pairs-days INT Download data of new pairs for given number of days. + Default: `None`. --timerange TIMERANGE Specify what timerange of data to use. --dl-trades Download trades instead of OHLCV data. The bot will @@ -48,10 +52,10 @@ optional arguments: exchange/pairs/timeframes. --data-format-ohlcv {json,jsongz,hdf5} Storage format for downloaded candle (OHLCV) data. - (default: `json`). + (default: `None`). --data-format-trades {json,jsongz,hdf5} Storage format for downloaded trades data. (default: - `jsongz`). + `None`). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/developer.md b/docs/developer.md index c09e528bf..d0731a233 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/p7nuUNVfP7) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions. ## Documentation diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 9cccfa93d..3a85aa885 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -1,5 +1,7 @@ # Using Freqtrade with Docker +This page explains how to run the bot with Docker. It is not meant to work out of the box. You'll still need to read through the documentation and understand how to properly configure it. + ## Install Docker Start by downloading and installing Docker CE for your platform: @@ -8,11 +10,11 @@ Start by downloading and installing Docker CE for your platform: * [Windows](https://docs.docker.com/docker-for-windows/install/) * [Linux](https://docs.docker.com/install/) -To simplify running freqtrade, please install [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start). +To simplify running freqtrade, [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start). ## Freqtrade with docker-compose -Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage. +Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/stable/docker-compose.yml) ready for usage. !!! Note - The following section assumes that `docker` and `docker-compose` are installed and available to the logged in user. @@ -20,7 +22,7 @@ Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.co ### Docker quick start -Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory. +Create a new directory and place the [docker-compose file](https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml) in this directory. === "PC/MAC/Linux" ``` bash @@ -46,6 +48,8 @@ Create a new directory and place the [docker-compose file](https://github.com/fr # Download the docker-compose file from the repository curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml + # Edit the compose file to use an image named `*_pi` (stable_pi or develop_pi) + # Pull the freqtrade image docker-compose pull @@ -63,6 +67,40 @@ Create a new directory and place the [docker-compose file](https://github.com/fr # image: freqtradeorg/freqtrade:develop_pi ``` +=== "ARM 64 Systenms (Mac M1, Raspberry Pi 4, Jetson Nano)" + In case of a Mac M1, make sure that your docker installation is running in native mode + Arm64 images are not yet provided via Docker Hub and need to be build locally first. + Depending on the device, this may take a few minutes (Apple M1) or multiple hours (Raspberry Pi) + + ``` bash + # Clone Freqtrade repository + git clone https://github.com/freqtrade/freqtrade.git + cd freqtrade + # Optionally switch to the stable version + git checkout stable + + # Modify your docker-compose file to enable building and change the image name + # (see the Note Box below for necessary changes) + + # Build image + docker-compose build + + # Create user directory structure + docker-compose run --rm freqtrade create-userdir --userdir user_data + + # Create configuration - Requires answering interactive questions + docker-compose run --rm freqtrade new-config --config user_data/config.json + ``` + + !!! Note "Change your docker Image" + You have to change the docker image in the docker-compose file for your arm64 build to work properly. + ``` yml + image: freqtradeorg/freqtrade:custom_arm64 + build: + context: . + dockerfile: "./docker/Dockerfile.aarch64" + ``` + The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image. The last 2 steps in the snippet create the directory with `user_data`, as well as (interactively) the default configuration based on your selections. @@ -75,7 +113,7 @@ The last 2 steps in the snippet create the directory with `user_data`, as well a 1. The configuration is now available as `user_data/config.json` 2. Copy a custom strategy to the directory `user_data/strategies/` -3. add the Strategy' class name to the `docker-compose.yml` file +3. Add the Strategy' class name to the `docker-compose.yml` file The `SampleStrategy` is run by default. @@ -90,6 +128,9 @@ Once this is done, you're ready to launch the bot in trading mode (Dry-run or Li docker-compose up -d ``` +!!! Warning "Default configuration" + While the configuration generated will be mostly functional, you will still need to verify that all options correspond to what you want (like Pricing, pairlist, ...) before starting the bot. + #### Monitoring the bot You can check for running instances with `docker-compose ps`. @@ -151,8 +192,8 @@ Head over to the [Backtesting Documentation](backtesting.md) to learn more. ### Additional dependencies with docker-compose -If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host. -For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) for an example). +If your strategy requires dependencies not included in the default image - it will be necessary to build the image on your host. +For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.custom](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.custom) for an example). You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions. diff --git a/docs/edge.md b/docs/edge.md index 5565ca2f9..237ff36f6 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,9 +1,9 @@ # Edge positioning -The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss. +The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss. !!! Warning - `Edge positioning` is not compatible with dynamic (volume-based) whitelist. + WHen using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data. !!! Note `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file. @@ -14,7 +14,7 @@ The `Edge Positioning` module uses probability to calculate your win rate and ri Trading strategies are not perfect. They are frameworks that are susceptible to the market and its indicators. Because the market is not at all predictable, sometimes a strategy will win and sometimes the same strategy will lose. -To obtain an edge in the market, a strategy has to make more money than it loses. Making money in trading is not only about *how often* the strategy makes or loses money. +To obtain an edge in the market, a strategy has to make more money than it loses. Making money in trading is not only about *how often* the strategy makes or loses money. !!! tip "It doesn't matter how often, but how much!" A bad strategy might make 1 penny in *ten* transactions but lose 1 dollar in *one* transaction. If one only checks the number of winning trades, it would be misleading to think that the strategy is actually making a profit. @@ -215,16 +215,20 @@ Let's say the stake currency is **ETH** and there is $10$ **ETH** on the wallet. usage: freqtrade edge [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TIMEFRAME] [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] - [--fee FLOAT] [--stoplosses STOPLOSS_RANGE] + [--fee FLOAT] [-p PAIRS [PAIRS ...]] + [--stoplosses STOPLOSS_RANGE] optional arguments: -h, --help show this help message and exit -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME - Specify ticker interval (`1m`, `5m`, `30m`, `1h`, - `1d`). + Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`). --timerange TIMERANGE Specify what timerange of data to use. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `None`). --max-open-trades INT Override the value of the `max_open_trades` configuration setting. @@ -233,6 +237,9 @@ optional arguments: setting. --fee FLOAT Specify fee ratio. Will be applied twice (on trade entry and exit). + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Limit command to these pairs. Pairs are space- + separated. --stoplosses STOPLOSS_RANGE Defines a range of stoploss values against which edge will assess the strategy. The format is "min,max,step" diff --git a/docs/exchanges.md b/docs/exchanges.md index d877e6da2..e54f97714 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -7,18 +7,17 @@ This page combines common gotchas and informations which are exchange-specific a !!! Tip "Stoploss on Exchange" Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. -### Blacklists +### Binance Blacklist For Binance, please add `"BNB/"` to your blacklist to avoid issues. -Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB order unsellable as the expected amount is not there anymore. +Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB trade unsellable as the expected amount is not there anymore. ### Binance sites -Binance has been split into 3, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. +Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. * [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`. * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`. -* [binance.je](https://www.binance.je/) - Binance Jersey, trading fiat currencies. Use exchange id: `binanceje`. ## Kraken @@ -40,12 +39,23 @@ Due to the heavy rate-limiting applied by Kraken, the following configuration se }, ``` +!!! Warning "Downloading data from kraken" + Downloading kraken data will require significantly more memory (RAM) than any other exchange, as the trades-data needs to be converted into candles on your machine. + It will also take a long time, as freqtrade will need to download every single trade that happened on the exchange for the pair / timerange combination, therefore please be patient. + +!!! Warning "rateLimit tuning" + Please pay attention that rateLimit configuration entry holds delay in milliseconds between requests, NOT requests\sec rate. + So, in order to mitigate Kraken API "Rate limit exceeded" exception, this configuration should be increased, NOT decreased. + ## Bittrex ### Order types Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faq.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy). +Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment. +Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected. + ### Restricted markets Bittrex split its exchange into US and International versions. @@ -92,8 +102,22 @@ To use subaccounts with FTX, you need to edit the configuration and add the foll } ``` -!!! Note - Older versions of freqtrade may require this key to be added to `"ccxt_async_config"` as well. +## Kucoin + +Kucoin requries a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows: + +```json +"exchange": { + "name": "kucoin", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "password": "your_exchange_api_key_password", +``` + +### Kucoin Blacklists + +For Kucoin, please add `"KCS/"` to your blacklist to avoid issues. +Accounts having KCS accounts use this to pay for fees - if your first trade happens to be on `KCS`, further trades will consume this position and make the initial KCS trade unsellable as the expected amount is not there anymore. ## All exchanges @@ -117,3 +141,23 @@ Whether your exchange returns incomplete candles or not can be checked using [th Due to the danger of repainting, Freqtrade does not allow you to use this incomplete candle. However, if it is based on the need for the latest price for your strategy - then this requirement can be acquired using the [data provider](strategy-customization.md#possible-options-for-dataprovider) from within the strategy. + +### Advanced Freqtrade Exchange configuration + +Advanced options can be configured using the `_ft_has_params` setting, which will override Defaults and exchange-specific behavior. + +Available options are listed in the exchange-class as `_ft_has_default`. + +For example, to test the order type `FOK` with Kraken, and modify candle limit to 200 (so you only get 200 candles per API call): + +```json +"exchange": { + "name": "kraken", + "_ft_has_params": { + "order_time_in_force": ["gtc", "fok"], + "ohlcv_candle_limit": 200 + } +``` + +!!! Warning + Please make sure to fully understand the impacts of these settings before modifying them. diff --git a/docs/faq.md b/docs/faq.md index 8a0c61b29..e5da550fd 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,5 +1,19 @@ # Freqtrade FAQ +## Supported Markets + +Freqtrade supports spot trading only. + +### Can I open short positions? + +No, Freqtrade does not support trading with margin / leverage, and cannot open short positions. + +In some cases, your exchange may provide leveraged spot tokens which can be traded with Freqtrade eg. BTCUP/USD, BTCDOWN/USD, ETHBULL/USD, ETHBEAR/USD, etc... + +### Can I trade options or futures? + +No, options and futures trading are not supported. + ## Beginner Tips & Tricks * When you work with your strategy & hyperopt file you should use a proper code editor like VSCode or PyCharm. A good code editor will provide syntax highlighting as well as line numbers, making it easy to find syntax errors (most likely pointed out by Freqtrade during startup). @@ -38,12 +52,11 @@ you can't say much from few trades. ### I’d like to make changes to the config. Can I do that without having to kill the bot? -Yes. You can edit your config, use the `/stop` command in Telegram, followed by `/reload_config` and the bot will run with the new config. +Yes. You can edit your config and use the `/reload_config` command to reload the configuration. The bot will stop, reload the configuration and strategy and will restart with the new configuration and strategy. ### I want to improve the bot with a new strategy -That's great. We have a nice backtesting and hyperoptimization setup. See -the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-commands). +That's great. We have a nice backtesting and hyperoptimization setup. See the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-commands). ### Is there a setting to only SELL the coins being held and not perform anymore BUYS? @@ -143,7 +156,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD ### Why does it take a long time to run hyperopt? -* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. +* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/p7nuUNVfP7). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers: diff --git a/docs/hyperopt.md b/docs/hyperopt.md index ec155062f..a117ac1ce 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -1,19 +1,22 @@ # Hyperopt This page explains how to tune your strategy by finding the optimal -parameters, a process called hyperparameter optimization. The bot uses several -algorithms included in the `scikit-optimize` package to accomplish this. The -search will burn all your CPU cores, make your laptop sound like a fighter jet -and still take a long time. +parameters, a process called hyperparameter optimization. The bot uses algorithms included in the `scikit-optimize` package to accomplish this. +The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time. In general, the search for best parameters starts with a few random combinations (see [below](#reproducible-results) for more details) and then uses Bayesian search with a ML regressor algorithm (currently ExtraTreesRegressor) to quickly find a combination of parameters in the search hyperspace that minimizes the value of the [loss function](#loss-functions). -Hyperopt requires historic data to be available, just as backtesting does. +Hyperopt requires historic data to be available, just as backtesting does (hyperopt runs backtesting many times with different parameters). To learn how to get data for the pairs and exchange you're interested in, head over to the [Data Downloading](data-download.md) section of the documentation. !!! Bug Hyperopt can crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133) +!!! Note + Since 2021.4 release you no longer have to write a separate hyperopt class, but can configure the parameters directly in the strategy. + The legacy method is still supported, but it is no longer the recommended way of setting up hyperopt. + The legacy documentation is available at [Legacy Hyperopt](advanced-hyperopt.md#legacy-hyperopt). + ## Install hyperopt dependencies Since Hyperopt dependencies are not needed to run the bot itself, are heavy, can not be easily built on some platforms (like Raspberry PI), they are not installed by default. Before you run Hyperopt, you need to install the corresponding dependencies, as described in this section below. @@ -34,7 +37,6 @@ pip install -r requirements-hyperopt.txt ## Hyperopt command reference - ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] @@ -42,8 +44,10 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] - [--hyperopt NAME] [--hyperopt-path PATH] [--eps] - [--dmmp] [--enable-protections] [-e INT] + [-p PAIRS [PAIRS ...]] [--hyperopt NAME] + [--hyperopt-path PATH] [--eps] [--dmmp] + [--enable-protections] + [--dry-run-wallet DRY_RUN_WALLET] [-e INT] [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] @@ -52,8 +56,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] optional arguments: -h, --help show this help message and exit -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME - Specify ticker interval (`1m`, `5m`, `30m`, `1h`, - `1d`). + Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`). --timerange TIMERANGE Specify what timerange of data to use. --data-format-ohlcv {json,jsongz,hdf5} @@ -67,6 +70,9 @@ optional arguments: setting. --fee FLOAT Specify fee ratio. Will be applied twice (on trade entry and exit). + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Limit command to these pairs. Pairs are space- + separated. --hyperopt NAME Specify hyperopt class name which will be used by the bot. --hyperopt-path PATH Specify additional lookup path for Hyperopt and @@ -82,6 +88,9 @@ optional arguments: Enable protections for backtesting.Will slow backtesting down by a considerable amount, but will include configured protections + --dry-run-wallet DRY_RUN_WALLET, --starting-balance DRY_RUN_WALLET + Starting balance, used for backtesting / hyperopt and + dry-runs. -e INT, --epochs INT Specify number of epochs (default: 100). --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...] Specify which parameters to hyperopt. Space-separated @@ -100,7 +109,8 @@ optional arguments: reproducible hyperopt results. --min-trades INT Set minimal desired number of trades for evaluations in the hyperopt optimization path (default: 1). - --hyperopt-loss NAME Specify the class name of the hyperopt loss function + --hyperopt-loss NAME, --hyperoptloss NAME + Specify the class name of the hyperopt loss function class (IHyperOptLoss). Different functions can generate completely different results, since the target for optimization is different. Built-in @@ -133,47 +143,19 @@ Strategy arguments: ``` -## Prepare Hyperopting - -Before we start digging into Hyperopt, we recommend you to take a look at -the sample hyperopt file located in [user_data/hyperopts/](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt.py). - -Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar. - -!!! Tip "About this page" - For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class. - -The simplest way to get started is to use the following, command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`. - -``` bash -freqtrade new-hyperopt --hyperopt AwesomeHyperopt -``` - ### Hyperopt checklist Checklist on all tasks / possibilities in hyperopt Depending on the space you want to optimize, only some of the below are required: -* fill `buy_strategy_generator` - for buy signal optimization -* fill `indicator_space` - for buy signal optimization -* fill `sell_strategy_generator` - for sell signal optimization -* fill `sell_indicator_space` - for sell signal optimization +* define parameters with `space='buy'` - for buy signal optimization +* define parameters with `space='sell'` - for sell signal optimization !!! Note - `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. + `populate_indicators` needs to create all indicators any of the spaces may use, otherwise hyperopt will not work. -Optional in hyperopt - can also be loaded from a strategy (recommended): - -* `populate_indicators` - fallback to create indicators -* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy -* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy - -!!! Note - You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods. - Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. - -Rarely you may also need to override: +Rarely you may also need to create a [nested class](advanced-hyperopt.md#overriding-pre-defined-spaces) named `HyperOpt` and implement * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default) * `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps) @@ -181,31 +163,30 @@ Rarely you may also need to override: * `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default) !!! Tip "Quickly optimize ROI, stoploss and trailing stoploss" - You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything (i.e. without creation of a "complete" Hyperopt class with dimensions, parameters, triggers and guards, as described in this document) from the default hyperopt template by relying on your strategy to do most of the calculations. + You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything in your strategy. - ```python + ``` bash # Have a working strategy at hand. - freqtrade new-hyperopt --hyperopt EmptyHyperopt - - freqtrade hyperopt --hyperopt EmptyHyperopt --hyperopt-loss SharpeHyperOptLossDaily --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100 + freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100 ``` -### Create a Custom Hyperopt File +### Hyperopt execution logic -Let assume you want a hyperopt file `AwesomeHyperopt.py`: +Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators. -``` bash -freqtrade new-hyperopt --hyperopt AwesomeHyperopt -``` +Hyperopt will then spawn into different processes (number of processors, or `-j `), and run backtesting over and over again, changing the parameters that are part of the `--spaces` defined. -This command will create a new hyperopt file from a template, allowing you to get started quickly. +For every new set of parameters, freqtrade will run first `populate_buy_trend()` followed by `populate_sell_trend()`, and then run the regular backtesting process to simulate trades. + +After backtesting, the results are passed into the [loss function](#loss-functions), which will evaluate if this result was better or worse than previous results. +Based on the loss function result, hyperopt will determine the next set of parameters to try in the next round of backtesting. ### Configure your Guards and Triggers -There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: +There are two places you need to change in your strategy file to add a new buy hyperopt for testing: -* Inside `indicator_space()` - the parameters hyperopt shall be optimizing. -* Within `buy_strategy_generator()` - populate the nested `populate_buy_trend()` to apply the parameters. +* Define the parameters at the class level hyperopt shall be optimizing. +* Within `populate_buy_trend()` - use defined parameter values instead of raw constants. There you have two different types of indicators: 1. `guards` and 2. `triggers`. @@ -217,100 +198,107 @@ There you have two different types of indicators: 1. `guards` and 2. `triggers`. However, this guide will make this distinction to make it clear that signals should not be "sticking". Sticking signals are signals that are active for multiple candles. This can lead into buying a signal late (right before the signal disappears - which means that the chance of success is a lot lower than right at the beginning). -Hyper-optimization will, for each epoch round, pick one trigger and possibly -multiple guards. The constructed strategy will be something like "*buy exactly when close price touches lower Bollinger band, BUT only if -ADX > 10*". - -If you have updated the buy strategy, i.e. changed the contents of `populate_buy_trend()` method, you have to update the `guards` and `triggers` your hyperopt must use correspondingly. +Hyper-optimization will, for each epoch round, pick one trigger and possibly multiple guards. #### Sell optimization Similar to the buy-signal above, sell-signals can also be optimized. Place the corresponding settings into the following methods -* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. -* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters. +* Define the parameters at the class level hyperopt shall be optimizing, either naming them `sell_*`, or by explicitly defining `space='sell'`. +* Within `populate_sell_trend()` - use defined parameter values instead of raw constants. The configuration and rules are the same than for buy signals. -To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. - -#### Using timeframe as a part of the Strategy - -The Strategy class exposes the timeframe value as the `self.timeframe` attribute. -The same value is available as class-attribute `HyperoptName.timeframe`. -In the case of the linked sample-value this would be `AwesomeHyperopt.timeframe`. ## Solving a Mystery -Let's say you are curious: should you use MACD crossings or lower Bollinger -Bands to trigger your buys. And you also wonder should you use RSI or ADX to -help with those buy decisions. If you decide to use RSI or ADX, which values -should I use for them? So let's use hyperparameter optimization to solve this -mystery. +Let's say you are curious: should you use MACD crossings or lower Bollinger Bands to trigger your buys. +And you also wonder should you use RSI or ADX to help with those buy decisions. +If you decide to use RSI or ADX, which values should I use for them? -We will start by defining a search space: +So let's use hyperparameter optimization to solve this mystery. -```python - def indicator_space() -> List[Dimension]: +### Defining indicators to be used + +We start by calculating the indicators our strategy is going to use. + +``` python +class MyAwesomeStrategy(IStrategy): + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Define your Hyperopt space for searching strategy parameters + Generate all indicators used by the strategy """ - return [ - Integer(20, 40, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal'], name='trigger') - ] + dataframe['adx'] = ta.ADX(dataframe) + dataframe['rsi'] = ta.RSI(dataframe) + macd = ta.MACD(dataframe) + dataframe['macd'] = macd['macd'] + dataframe['macdsignal'] = macd['macdsignal'] + dataframe['macdhist'] = macd['macdhist'] + + bollinger = ta.BBANDS(dataframe, timeperiod=20, nbdevup=2.0, nbdevdn=2.0) + dataframe['bb_lowerband'] = bollinger['lowerband'] + dataframe['bb_middleband'] = bollinger['middleband'] + dataframe['bb_upperband'] = bollinger['upperband'] + return dataframe ``` -Above definition says: I have five parameters I want you to randomly combine -to find the best combination. Two of them are integer values (`adx-value` -and `rsi-value`) and I want you test in the range of values 20 to 40. +### Hyperoptable parameters + +We continue to define hyperoptable parameters: + +```python +class MyAwesomeStrategy(IStrategy): + buy_adx = DecimalParameter(20, 40, decimals=1, default=30.1, space="buy") + buy_rsi = IntParameter(20, 40, default=30, space="buy") + buy_adx_enabled = CategoricalParameter([True, False], default=True, space="buy") + buy_rsi_enabled = CategoricalParameter([True, False], default=False, space="buy") + buy_trigger = CategoricalParameter(["bb_lower", "macd_cross_signal"], default="bb_lower", space="buy") +``` + +The above definition says: I have five parameters I want to randomly combine to find the best combination. +`buy_rsi` is an integer parameter, which will be tested between 20 and 40. This space has a size of 20. +`buy_adx` is a decimal parameter, which will be evaluated between 20 and 40 with 1 decimal place (so values are 20.1, 20.2, ...). This space has a size of 200. Then we have three category variables. First two are either `True` or `False`. -We use these to either enable or disable the ADX and RSI guards. The last -one we call `trigger` and use it to decide which buy trigger we want to use. +We use these to either enable or disable the ADX and RSI guards. +The last one we call `trigger` and use it to decide which buy trigger we want to use. + +!!! Note "Parameter space assignment" + Parameters must either be assigned to a variable named `buy_*` or `sell_*` - or contain `space='buy'` | `space='sell'` to be assigned to a space correctly. + If no parameter is available for a space, you'll receive the error that no space was found when running hyperopt. So let's write the buy strategy using these values: ```python - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + conditions = [] + # GUARDS AND TRENDS + if self.buy_adx_enabled.value: + conditions.append(dataframe['adx'] > self.buy_adx.value) + if self.buy_rsi_enabled.value: + conditions.append(dataframe['rsi'] < self.buy_rsi.value) - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) + # TRIGGERS + if self.buy_trigger.value == 'bb_lower': + conditions.append(dataframe['close'] < dataframe['bb_lowerband']) + if self.buy_trigger.value == 'macd_cross_signal': + conditions.append(qtpylib.crossed_above( + dataframe['macd'], dataframe['macdsignal'] + )) - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) + # Check that volume is not 0 + conditions.append(dataframe['volume'] > 0) - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 - return dataframe - - return populate_buy_trend + return dataframe ``` Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations. -It will use the given historical data and make buys based on the buy signals generated with the above function. +It will use the given historical data and simulate buys based on the buy signals generated with the above function. Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)). !!! Note @@ -318,6 +306,108 @@ Based on the results, hyperopt will tell you which parameter combination produce When you want to test an indicator that isn't used by the bot currently, remember to add it to the `populate_indicators()` method in your strategy or hyperopt file. +## Parameter types + +There are four parameter types each suited for different purposes. + +* `IntParameter` - defines an integral parameter with upper and lower boundaries of search space. +* `DecimalParameter` - defines a floating point parameter with a limited number of decimals (default 3). Should be preferred instead of `RealParameter` in most cases. +* `RealParameter` - defines a floating point parameter with upper and lower boundaries and no precision limit. Rarely used as it creates a space with a near infinite number of possibilities. +* `CategoricalParameter` - defines a parameter with a predetermined number of choices. + +!!! Tip "Disabling parameter optimization" + Each parameter takes two boolean parameters: + * `load` - when set to `False` it will not load values configured in `buy_params` and `sell_params`. + * `optimize` - when set to `False` parameter will not be included in optimization process. + Use these parameters to quickly prototype various ideas. + +!!! Warning + Hyperoptable parameters cannot be used in `populate_indicators` - as hyperopt does not recalculate indicators for each epoch, so the starting value would be used in this case. + +### Optimizing an indicator parameter + +Assuming you have a simple strategy in mind - a EMA cross strategy (2 Moving averages crossing) - and you'd like to find the ideal parameters for this strategy. + +``` python +from pandas import DataFrame +from functools import reduce + +import talib.abstract as ta + +from freqtrade.strategy import IStrategy +from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter +import freqtrade.vendor.qtpylib.indicators as qtpylib + +class MyAwesomeStrategy(IStrategy): + stoploss = -0.05 + timeframe = '15m' + # Define the parameter spaces + buy_ema_short = IntParameter(3, 50, default=5) + buy_ema_long = IntParameter(15, 200, default=50) + + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + """Generate all indicators used by the strategy""" + + # Calculate all ema_short values + for val in self.buy_ema_short.range: + dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val) + + # Calculate all ema_long values + for val in self.buy_ema_long.range: + dataframe[f'ema_long_{val}'] = ta.EMA(dataframe, timeperiod=val) + + return dataframe + + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + conditions = [] + conditions.append(qtpylib.crossed_above( + dataframe[f'ema_short_{self.buy_ema_short.value}'], dataframe[f'ema_long_{self.buy_ema_long.value}'] + )) + + # Check that volume is not 0 + conditions.append(dataframe['volume'] > 0) + + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'buy'] = 1 + return dataframe + + def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + conditions = [] + conditions.append(qtpylib.crossed_above( + dataframe[f'ema_long_{self.buy_ema_long.value}'], dataframe[f'ema_short_{self.buy_ema_short.value}'] + )) + + # Check that volume is not 0 + conditions.append(dataframe['volume'] > 0) + + if conditions: + dataframe.loc[ + reduce(lambda x, y: x & y, conditions), + 'sell'] = 1 + return dataframe +``` + +Breaking it down: + +Using `self.buy_ema_short.range` will return a range object containing all entries between the Parameters low and high value. +In this case (`IntParameter(3, 50, default=5)`), the loop would run for all numbers between 3 and 50 (`[3, 4, 5, ... 49, 50]`). +By using this in a loop, hyperopt will generate 48 new columns (`['buy_ema_3', 'buy_ema_4', ... , 'buy_ema_50']`). + +Hyperopt itself will then use the selected value to create the buy and sell signals + +While this strategy is most likely too simple to provide consistent profit, it should serve as an example how optimize indicator parameters. + +!!! Note + `self.buy_ema_short.range` will act differently between hyperopt and other modes. For hyperopt, the above example may generate 48 new columns, however for all other modes (backtesting, dry/live), it will only generate the column for the selected value. You should therefore avoid using the resulting column with explicit values (values other than `self.buy_ema_short.value`). + +??? Hint "Performance tip" + By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter. + While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values). + You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space. + ## Loss-functions Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results. @@ -339,16 +429,14 @@ Creation of a custom loss function is covered in the [Advanced Hyperopt](advance ## Execute Hyperopt Once you have updated your hyperopt configuration you can run it. -Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. More time usually results in better results. +Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -freqtrade hyperopt --config config.json --hyperopt --hyperopt-loss --strategy -e 500 --spaces all +freqtrade hyperopt --config config.json --hyperopt-loss --strategy -e 500 --spaces all ``` -Use `` as the name of the custom hyperopt used. - The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs. Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results. @@ -362,30 +450,23 @@ The `--spaces all` option determines that all possible parameters should be opti ### Execute Hyperopt with different historical data source If you would like to hyperopt parameters using an alternate historical data set that -you have on-disk, use the `--datadir PATH` option. By default, hyperopt -uses data from directory `user_data/data`. +you have on-disk, use the `--datadir PATH` option. By default, hyperopt uses data from directory `user_data/data`. ### Running Hyperopt with a smaller test-set Use the `--timerange` argument to change how much of the test-set you want to use. -For example, to use one month of data, pass the following parameter to the hyperopt call: +For example, to use one month of data, pass `--timerange 20210101-20210201` (from january 2021 - february 2021) to the hyperopt call. + +Full command: ```bash -freqtrade hyperopt --hyperopt --strategy --timerange 20180401-20180501 -``` - -### Running Hyperopt using methods from a strategy - -Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided. - -```bash -freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy +freqtrade hyperopt --hyperopt --strategy --timerange 20210101-20210201 ``` ### Running Hyperopt with Smaller Search Space Use the `--spaces` option to limit the search space used by hyperopt. -Letting Hyperopt optimize everything is a huuuuge search space. +Letting Hyperopt optimize everything is a huuuuge search space. Often it might make more sense to start by just searching for initial buy algorithm. Or maybe you just want to optimize your stoploss or roi table for that awesome new buy strategy you have. @@ -402,90 +483,48 @@ Legal values are: The default Hyperopt Search Space, used when no `--space` command line option is specified, does not include the `trailing` hyperspace. We recommend you to run optimization for the `trailing` hyperspace separately, when the best parameters for other hyperspaces were found, validated and pasted into your custom strategy. -### Position stacking and disabling max market positions - -In some situations, you may need to run Hyperopt (and Backtesting) with the -`--eps`/`--enable-position-staking` and `--dmmp`/`--disable-max-market-positions` arguments. - -By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one -open trade is allowed for every traded pair. The total number of trades open for all pairs -is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to -some potential trades to be hidden (or masked) by previously open trades. - -The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times, -while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades` -during Hyperopt/Backtesting (which is equal to setting `max_open_trades` to a very high -number). - -!!! Note - Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality. - -You can also enable position stacking in the configuration file by explicitly setting -`"position_stacking"=true`. - -### Reproducible results - -The search for optimal parameters starts with a few (currently 30) random combinations in the hyperspace of parameters, random Hyperopt epochs. These random epochs are marked with an asterisk character (`*`) in the first column in the Hyperopt output. - -The initial state for generation of these random values (random state) is controlled by the value of the `--random-state` command line option. You can set it to some arbitrary value of your choice to obtain reproducible results. - -If you have not set this value explicitly in the command line options, Hyperopt seeds the random state with some random value for you. The random state value for each Hyperopt run is shown in the log, so you can copy and paste it into the `--random-state` command line option to repeat the set of the initial random epochs used. - -If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyper-optimization results with same random state value used. - ## Understand the Hyperopt Result -Once Hyperopt is completed you can use the result to create a new strategy. +Once Hyperopt is completed you can use the result to update your strategy. Given the following result from hyperopt: ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 -Buy hyperspace params: -{ 'adx-value': 44, - 'rsi-value': 29, - 'adx-enabled': False, - 'rsi-enabled': True, - 'trigger': 'bb_lower'} + # Buy hyperspace params: + buy_params = { + 'buy_adx': 44, + 'buy_rsi': 29, + 'buy_adx_enabled': False, + 'buy_rsi_enabled': True, + 'buy_trigger': 'bb_lower' + } ``` You should understand this result like: -- The buy trigger that worked best was `bb_lower`. -- You should not use ADX because `adx-enabled: False`) -- You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`) +* The buy trigger that worked best was `bb_lower`. +* You should not use ADX because `'buy_adx_enabled': False`. +* You should **consider** using the RSI indicator (`'buy_rsi_enabled': True`) and the best value is `29.0` (`'buy_rsi': 29.0`) -You have to look inside your strategy file into `buy_strategy_generator()` -method, what those values match to. +Your strategy class can immediately take advantage of these results. Simply copy hyperopt results block and paste them at class level, replacing old parameters (if any). New parameters will automatically be loaded next time strategy is executed. -So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block: +Transferring your whole hyperopt result to your strategy would then look like: ```python -(dataframe['rsi'] < 29.0) +class MyAwesomeStrategy(IStrategy): + # Buy hyperspace params: + buy_params = { + 'buy_adx': 44, + 'buy_rsi': 29, + 'buy_adx_enabled': False, + 'buy_rsi_enabled': True, + 'buy_trigger': 'bb_lower' + } ``` -Translating your whole hyperopt result as the new buy-signal would then look like: - -```python -def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - dataframe.loc[ - ( - (dataframe['rsi'] < 29.0) & # rsi-value - dataframe['close'] < dataframe['bb_lowerband'] # trigger - ), - 'buy'] = 1 - return dataframe -``` - -By default, hyperopt prints colorized results -- epochs with positive profit are printed in the green color. This highlighting helps you find epochs that can be interesting for later analysis. Epochs with zero total profit or with negative profits (losses) are printed in the normal color. If you do not need colorization of results (for instance, when you are redirecting hyperopt output to a file) you can switch colorization off by specifying the `--no-color` option in the command line. - -You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option. - -!!! Note "Windows and color output" - Windows does not support color-output natively, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL. - ### Understand Hyperopt ROI results If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table: @@ -493,13 +532,15 @@ If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'de ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 -ROI table: -{ 0: 0.10674, - 21: 0.09158, - 78: 0.03634, - 118: 0} + # ROI table: + minimal_roi = { + 0: 0.10674, + 21: 0.09158, + 78: 0.03634, + 118: 0 + } ``` In order to use this best ROI table found by Hyperopt in backtesting and for live trades/dry-run, copy-paste it as the value of the `minimal_roi` attribute of your custom strategy: @@ -519,23 +560,26 @@ As stated in the comment, you can also use it as the value of the `minimal_roi` #### Default ROI Search Space -If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the timeframe used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 5 digits after the decimal point): +If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the timeframe used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 3 digits after the decimal point): -| # step | 1m | | 5m | | 1h | | 1d | | -| ------ | ------ | ----------------- | -------- | ----------- | ---------- | ----------------- | ------------ | ----------------- | -| 1 | 0 | 0.01161...0.11992 | 0 | 0.03...0.31 | 0 | 0.06883...0.71124 | 0 | 0.12178...1.25835 | -| 2 | 2...8 | 0.00774...0.04255 | 10...40 | 0.02...0.11 | 120...480 | 0.04589...0.25238 | 2880...11520 | 0.08118...0.44651 | -| 3 | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 | -| 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 | +| # step | 1m | | 5m | | 1h | | 1d | | +| ------ | ------ | ------------- | -------- | ----------- | ---------- | ------------- | ------------ | ------------- | +| 1 | 0 | 0.011...0.119 | 0 | 0.03...0.31 | 0 | 0.068...0.711 | 0 | 0.121...1.258 | +| 2 | 2...8 | 0.007...0.042 | 10...40 | 0.02...0.11 | 120...480 | 0.045...0.252 | 2880...11520 | 0.081...0.446 | +| 3 | 4...20 | 0.003...0.015 | 20...100 | 0.01...0.04 | 240...1200 | 0.022...0.091 | 5760...28800 | 0.040...0.162 | +| 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 | These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used. If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default. -Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). +Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). A sample for these methods can be found in [sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +!!! Note "Reduced search space" + To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. + ### Understand Hyperopt Stoploss results If you are optimizing stoploss values (i.e. if optimization search-space contains 'all', 'default' or 'stoploss'), your result will look as follows and include stoploss: @@ -543,15 +587,18 @@ If you are optimizing stoploss values (i.e. if optimization search-space contain ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 -Buy hyperspace params: -{ 'adx-value': 44, - 'rsi-value': 29, - 'adx-enabled': False, - 'rsi-enabled': True, - 'trigger': 'bb_lower'} -Stoploss: -0.27996 + # Buy hyperspace params: + buy_params = { + 'buy_adx': 44, + 'buy_rsi': 29, + 'buy_adx_enabled': False, + 'buy_rsi_enabled': True, + 'buy_trigger': 'bb_lower' + } + + stoploss: -0.27996 ``` In order to use this best stoploss value found by Hyperopt in backtesting and for live trades/dry-run, copy-paste it as the value of the `stoploss` attribute of your custom strategy: @@ -572,6 +619,9 @@ If you have the `stoploss_space()` method in your custom hyperopt file, remove i Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +!!! Note "Reduced search space" + To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. + ### Understand Hyperopt Trailing Stop results If you are optimizing trailing stop values (i.e. if optimization search-space contains 'all' or 'trailing'), your result will look as follows and include trailing stop parameters: @@ -579,13 +629,13 @@ If you are optimizing trailing stop values (i.e. if optimization search-space co ``` Best result: - 45/100: 606 trades. Avg profit 1.04%. Total profit 0.31555614 BTC ( 630.48Σ%). Avg duration 150.3 mins. Objective: -1.10161 + 45/100: 606 trades. Avg profit 1.04%. Total profit 0.31555614 BTC ( 630.48%). Avg duration 150.3 mins. Objective: -1.10161 -Trailing stop: -{ 'trailing_only_offset_is_reached': True, - 'trailing_stop': True, - 'trailing_stop_positive': 0.02001, - 'trailing_stop_positive_offset': 0.06038} + # Trailing stop: + trailing_stop = True + trailing_stop_positive = 0.02001 + trailing_stop_positive_offset = 0.06038 + trailing_only_offset_is_reached = True ``` In order to use these best trailing stop parameters found by Hyperopt in backtesting and for live trades/dry-run, copy-paste them as the values of the corresponding attributes of your custom strategy: @@ -607,6 +657,59 @@ If you are optimizing trailing stop values, Freqtrade creates the 'trailing' opt Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +!!! Note "Reduced search space" + To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. + +### Reproducible results + +The search for optimal parameters starts with a few (currently 30) random combinations in the hyperspace of parameters, random Hyperopt epochs. These random epochs are marked with an asterisk character (`*`) in the first column in the Hyperopt output. + +The initial state for generation of these random values (random state) is controlled by the value of the `--random-state` command line option. You can set it to some arbitrary value of your choice to obtain reproducible results. + +If you have not set this value explicitly in the command line options, Hyperopt seeds the random state with some random value for you. The random state value for each Hyperopt run is shown in the log, so you can copy and paste it into the `--random-state` command line option to repeat the set of the initial random epochs used. + +If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyper-optimization results with same random state value used. + +## Output formatting + +By default, hyperopt prints colorized results -- epochs with positive profit are printed in the green color. This highlighting helps you find epochs that can be interesting for later analysis. Epochs with zero total profit or with negative profits (losses) are printed in the normal color. If you do not need colorization of results (for instance, when you are redirecting hyperopt output to a file) you can switch colorization off by specifying the `--no-color` option in the command line. + +You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option. + +!!! Note "Windows and color output" + Windows does not support color-output natively, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL. + +## Position stacking and disabling max market positions + +In some situations, you may need to run Hyperopt (and Backtesting) with the +`--eps`/`--enable-position-staking` and `--dmmp`/`--disable-max-market-positions` arguments. + +By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one +open trade is allowed for every traded pair. The total number of trades open for all pairs +is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to +some potential trades to be hidden (or masked) by previously open trades. + +The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times, +while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades` +during Hyperopt/Backtesting (which is equal to setting `max_open_trades` to a very high +number). + +!!! Note + Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality. + +You can also enable position stacking in the configuration file by explicitly setting +`"position_stacking"=true`. + +## Out of Memory errors + +As hyperopt consumes a lot of memory (the complete data needs to be in memory once per parallel backtesting process), it's likely that you run into "out of memory" errors. +To combat these, you have multiple options: + +* reduce the amount of pairs +* reduce the timerange used (`--timerange `) +* reduce the number of parallel processes (`-j `) +* Increase the memory of your machine + ## Show details of Hyperopt results After you run Hyperopt for the desired amount of epochs, you can later list all results for analysis, select only best or profitable once, and show the details for any of the epochs previously evaluated. This can be done with the `hyperopt-list` and `hyperopt-show` sub-commands. The usage of these sub-commands is described in the [Utils](utils.md#list-hyperopt-results) chapter. diff --git a/docs/images/logo.png b/docs/images/logo.png index c7138e84b..8a7ffdd70 100644 Binary files a/docs/images/logo.png and b/docs/images/logo.png differ diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 2653406e7..f19c5a181 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -4,7 +4,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade. In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). -Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. +Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler. @@ -29,6 +29,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`ShuffleFilter`](#shufflefilter) * [`SpreadFilter`](#spreadfilter) * [`RangeStabilityFilter`](#rangestabilityfilter) +* [`VolatilityFilter`](#volatilityfilter) !!! Tip "Testing pairlists" Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly. @@ -59,6 +60,8 @@ When used in the chain of Pairlist Handlers in a non-leading position (after Sta When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top assets from all available markets (with matching stake-currency) on the exchange. The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). +The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists. +Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data. `VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library: @@ -89,6 +92,7 @@ This filter allows freqtrade to ignore pairs until they have been listed for at #### PerformanceFilter Sorts pairs by past trade performance, as follows: + 1. Positive performance. 2. No closed trades yet. 3. Negative performance. @@ -108,6 +112,7 @@ The `PriceFilter` allows filtering of pairs by price. Currently the following pr * `min_price` * `max_price` +* `max_value` * `low_price_ratio` The `min_price` setting removes pairs where the price is below the specified price. This is useful if you wish to avoid trading very low-priced pairs. @@ -116,6 +121,11 @@ This option is disabled by default, and will only apply if set to > 0. The `max_price` setting removes pairs where the price is above the specified price. This is useful if you wish to trade only low-priced pairs. This option is disabled by default, and will only apply if set to > 0. +The `max_value` setting removes pairs where the minimum value change is above a specified value. +This is useful when an exchange has unbalanced limits. For example, if step-size = 1 (so you can only buy 1, or 2, or 3, but not 1.1 Coins) - and the price is pretty high (like 20\$) as the coin has risen sharply since the last limit adaption. +As a result of the above, you can only buy for 20\$, or 40\$ - but not for 25\$. +On exchanges that deduct fees from the receiving currency (e.g. FTX) - this can result in high value coins / amounts that are unsellable as the amount is slightly below the limit. + The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio. This option is disabled by default, and will only apply if set to > 0. @@ -164,9 +174,32 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit !!! Tip This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit. +#### VolatilityFilter + +Volatility is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatility of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)). + +This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. + +This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. + +In the below example: +If the volatility over the last 10 days is not in the range of 0.05-0.50, remove the pair from the whitelist. The filter is applied every 24h. + +```json +"pairlists": [ + { + "method": "VolatilityFilter", + "lookback_days": 10, + "min_volatility": 0.05, + "max_volatility": 0.50, + "refresh_period": 86400 + } +] +``` + ### Full example of Pairlist Handlers -The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value. +The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#pricefilter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value. ```json "exchange": { @@ -177,7 +210,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, { "method": "VolumePairList", "number_assets": 20, - "sort_key": "quoteVolume", + "sort_key": "quoteVolume" }, {"method": "AgeFilter", "min_days_listed": 10}, {"method": "PrecisionFilter"}, @@ -189,6 +222,13 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, "min_rate_of_change": 0.01, "refresh_period": 1440 }, + { + "method": "VolatilityFilter", + "lookback_days": 10, + "min_volatility": 0.05, + "max_volatility": 0.50, + "refresh_period": 86400 + }, {"method": "ShuffleFilter", "seed": 42} ], ``` diff --git a/docs/includes/pricing.md b/docs/includes/pricing.md index d8a72cc58..bdf27eb20 100644 --- a/docs/includes/pricing.md +++ b/docs/includes/pricing.md @@ -103,6 +103,10 @@ A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting When not using orderbook (`ask_strategy.use_order_book=False`), the price at the `ask_strategy.price_side` side (defaults to `"ask"`) from the ticker will be used as the sell price. +When not using orderbook (`ask_strategy.use_order_book=False`), Freqtrade uses the best `side` price from the ticker if it's below the `last` traded price from the ticker. Otherwise (when the `side` price is above the `last` price), it calculates a rate between `side` and `last` price. + +The `ask_strategy.bid_last_balance` configuration parameter controls this. A value of `0.0` will use `side` price, while `1.0` will use the last price and values between those interpolate between `side` and last price. + ### Market order pricing When using market orders, prices should be configured to use the "correct" side of the orderbook to allow realistic pricing detection. diff --git a/docs/includes/protections.md b/docs/includes/protections.md index de34383ac..6bc57153e 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -40,7 +40,9 @@ All protection end times are rounded up to the next candle to avoid sudden, unex #### Stoploss Guard -`StoplossGuard` selects all trades within `lookback_period` in minutes (or in candles when using `lookback_period_candles`), and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration` in minutes (or in candles when using `stop_duration_candles`). +`StoplossGuard` selects all trades within `lookback_period` in minutes (or in candles when using `lookback_period_candles`). +If `trade_limit` or more trades resulted in stoploss, trading will stop for `stop_duration` in minutes (or in candles when using `stop_duration_candles`). + This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. diff --git a/docs/index.md b/docs/index.md index b489861f0..871172cfc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,16 +1,13 @@ -# Freqtrade +![freqtrade](assets/freqtrade_poweredby.svg) + [![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/) [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) [![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability) Star - Fork - Download - -Follow @freqtrade ## Introduction @@ -35,6 +32,22 @@ Freqtrade is a crypto-currency algorithmic trading software developed in python - Control/Monitor: Use Telegram or a REST API (start/stop the bot, show profit/loss, daily summary, current open trades results, etc.). - Analyse: Further analysis can be performed on either Backtesting data or Freqtrade trading history (SQL database), including automated standard plots, and methods to load the data into [interactive environments](data-analysis.md). +## Supported exchange marketplaces + +Please read the [exchange specific notes](exchanges.md) to learn about eventual, special configurations needed for each exchange. + +- [X] [Binance](https://www.binance.com/) ([*Note for binance users](exchanges.md#blacklists)) +- [X] [Bittrex](https://bittrex.com/) +- [X] [FTX](https://ftx.com) +- [X] [Kraken](https://kraken.com/) +- [ ] [potentially many others through ccxt](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ + +### Community tested + +Exchanges confirmed working by the community: + +- [X] [Bitvavo](https://bitvavo.com/) + ## Requirements ### Hardware requirements @@ -63,9 +76,9 @@ Alternatively For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. -Please check out our [discord server](https://discord.gg/MA9v74M). +Please check out our [discord server](https://discord.gg/p7nuUNVfP7). -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-l9d9iqgl-9cVBIeBkCBa8j6upSmd_NA). +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). ## Ready to try? diff --git a/docs/installation.md b/docs/installation.md index d2661f88f..c19965a18 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -60,7 +60,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces sudo apt-get update # install packages - sudo apt install -y python3-pip python3-venv python3-pandas python3-pip git + sudo apt install -y python3-pip python3-venv python3-pandas git ``` === "RaspberryPi/Raspbian" @@ -269,7 +269,7 @@ git clone https://github.com/freqtrade/freqtrade.git cd freqtrade ``` -#### Freqtrade instal: Conda Environment +#### Freqtrade install: Conda Environment Prepare conda-freqtrade environment, using file `environment.yml`, which exist in main freqtrade directory diff --git a/docs/partials/header.html b/docs/partials/header.html index f5243225b..22132bc96 100644 --- a/docs/partials/header.html +++ b/docs/partials/header.html @@ -6,22 +6,22 @@ This file was automatically generated - do not edit {% set site_url = site_url ~ "/index.html" %} {% endif %}
- - - - + + +
diff --git a/docs/plotting.md b/docs/plotting.md index 19ddb4f57..9fae38504 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -37,7 +37,7 @@ usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] - Show profits for only these pairs. Pairs are space- + Limit command to these pairs. Pairs are space- separated. --indicators1 INDICATORS1 [INDICATORS1 ...] Set indicators from your strategy you want in the @@ -66,8 +66,7 @@ optional arguments: --timerange TIMERANGE Specify what timerange of data to use. -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME - Specify ticker interval (`1m`, `5m`, `30m`, `1h`, - `1d`). + Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`). --no-trades Skip using trades from backtesting file and DB. Common arguments: @@ -91,6 +90,7 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. + ``` Example: @@ -170,9 +170,15 @@ Additional features when using plot_config include: * Specify additional subplots * Specify indicator pairs to fill area in between -The sample plot configuration below specifies fixed colors for the indicators. Otherwise consecutive plots may produce different colorschemes each time, making comparisons difficult. +The sample plot configuration below specifies fixed colors for the indicators. Otherwise, consecutive plots may produce different color schemes each time, making comparisons difficult. It also allows multiple subplots to display both MACD and RSI at the same time. +Plot type can be configured using `type` key. Possible types are: +* `scatter` corresponding to `plotly.graph_objects.Scatter` class (default). +* `bar` corresponding to `plotly.graph_objects.Bar` class. + +Extra parameters to `plotly.graph_objects.*` constructor can be specified in `plotly` dict. + Sample configuration with inline comments explaining the process: ``` python @@ -188,7 +194,7 @@ Sample configuration with inline comments explaining the process: 'senkou_a': { 'color': 'green', #optional 'fill_to': 'senkou_b', - 'fill_label': 'Ichimoku Cloud' #optional, + 'fill_label': 'Ichimoku Cloud', #optional 'fill_color': 'rgba(255,76,46,0.2)', #optional }, # plot senkou_b, too. Not only the area to it. @@ -198,7 +204,8 @@ Sample configuration with inline comments explaining the process: # Create subplot MACD "MACD": { 'macd': {'color': 'blue', 'fill_to': 'macdhist'}, - 'macdsignal': {'color': 'orange'} + 'macdsignal': {'color': 'orange'}, + 'macdhist': {'type': 'bar', 'plotly': {'opacity': 0.9}} }, # Additional subplot RSI "RSI": { @@ -213,6 +220,9 @@ Sample configuration with inline comments explaining the process: The above configuration assumes that `ema10`, `ema50`, `senkou_a`, `senkou_b`, `macd`, `macdsignal`, `macdhist` and `rsi` are columns in the DataFrame created by the strategy. +!!! Warning + `plotly` arguments are only supported with plotly library and will not work with freq-ui. + ## Plot profit ![plot-profit](assets/plot-profit.png) @@ -245,7 +255,7 @@ usage: freqtrade plot-profit [-h] [-v] [--logfile FILE] [-V] [-c PATH] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] - Show profits for only these pairs. Pairs are space- + Limit command to these pairs. Pairs are space- separated. --timerange TIMERANGE Specify what timerange of data to use. @@ -264,8 +274,8 @@ optional arguments: Specify the source for trades (Can be DB or file (backtest file)) Default: file -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME - Specify ticker interval (`1m`, `5m`, `30m`, `1h`, - `1d`). + Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`). + --auto-open Automatically open generated plot. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -288,6 +298,7 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. + ``` The `-p/--pairs` argument, can be used to limit the pairs that are considered for this calculation. diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 94b2fca39..776d5592a 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,4 @@ -mkdocs-material==6.2.8 +mkdocs==1.2 +mkdocs-material==7.1.7 mdx_truly_sane_lists==1.2 -pymdown-extensions==8.1.1 +pymdown-extensions==8.2 diff --git a/docs/rest-api.md b/docs/rest-api.md index e2b94f080..b9b2b29be 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -71,7 +71,10 @@ If you run your bot using docker, you'll need to have the bot listen to incoming "api_server": { "enabled": true, "listen_ip_address": "0.0.0.0", - "listen_port": 8080 + "listen_port": 8080, + "username": "Freqtrader", + "password": "SuperSecret1!", + //... }, ``` @@ -106,7 +109,10 @@ By default, the script assumes `127.0.0.1` (localhost) and port `8080` to be use "api_server": { "enabled": true, "listen_ip_address": "0.0.0.0", - "listen_port": 8080 + "listen_port": 8080, + "username": "Freqtrader", + "password": "SuperSecret1!", + //... } } ``` @@ -124,13 +130,15 @@ python3 scripts/rest_client.py --config rest_config.json [optional par | `stop` | Stops the trader. | `stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules. | `reload_config` | Reloads the configuration file. -| `trades` | List last trades. +| `trades` | List last trades. Limited to 500 trades per call. +| `trade/` | Get specific trade. | `delete_trade ` | Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange. | `show_config` | Shows part of the current configuration with relevant settings to operation. | `logs` | Shows last log messages. | `status` | Lists all open trades. | `count` | Displays number of trades used and available. | `locks` | Displays currently locked pairs. +| `delete_lock ` | Deletes (disables) the lock by id. | `profit` | Display a summary of your profit/loss from close trades and some stats about your performance. | `forcesell ` | Instantly sells the given trade (Ignoring `minimum_roi`). | `forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`). @@ -180,7 +188,12 @@ count Return the amount of open trades. daily - Return the amount of open trades. + Return the profits for each day, and amount of trades. + +delete_lock + Delete (disable) lock from the database. + + :param lock_id: ID for the lock to delete delete_trade Delete trade from the database. @@ -202,10 +215,13 @@ forcesell :param tradeid: Id of the trade (can be received via status command) +locks + Return current locks + logs Show latest logs. - :param limit: Limits log messages to the last logs. No limit to get all the trades. + :param limit: Limits log messages to the last logs. No limit to get the entire log. pair_candles Return live dataframe for . @@ -225,6 +241,9 @@ pair_history performance Return the performance of the different coins. +ping + simple ping + plot_config Return plot configuration if the strategy defines one. @@ -261,17 +280,22 @@ strategy :param strategy: Strategy class name -trades - Return trades history. +trade + Return specific trade - :param limit: Limits trades to the X last trades. No limit to get all the trades. + :param trade_id: Specify which trade to get. + +trades + Return trades history, sorted by id + + :param limit: Limits trades to the X last trades. Max 500 trades. + :param offset: Offset by this amount of trades. version Return the version of the bot. whitelist Show the current whitelist. - ``` ### OpenAPI interface diff --git a/docs/sandbox-testing.md b/docs/sandbox-testing.md index 9c14412de..5f572eba8 100644 --- a/docs/sandbox-testing.md +++ b/docs/sandbox-testing.md @@ -6,6 +6,10 @@ With some configuration, freqtrade (in combination with ccxt) provides access to This document is an overview to configure Freqtrade to be used with sandboxes. This can be useful to developers and trader alike. +!!! Warning + Sandboxes usually have very low volume, and either a very wide spread, or no orders available at all. + Therefore, sandboxes will usually not do a good job of showing you how a strategy would work in real trading. + ## Exchanges known to have a sandbox / testnet * [binance](https://testnet.binance.vision/) diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 569af33ff..477396931 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -19,7 +19,7 @@ The freqtrade docker image does contain sqlite3, so you can edit the database wi ``` bash docker-compose exec freqtrade /bin/bash -sqlite3 .sqlite +sqlite3 .sqlite ``` ## Open the DB @@ -99,3 +99,32 @@ DELETE FROM trades WHERE id = 31; !!! Warning This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause. + +## Use a different database system + +!!! Warning + By using one of the below database systems, you acknowledge that you know how to manage such a system. Freqtrade will not provide any support with setup or maintenance (or backups) of the below database systems. + +### PostgreSQL + +Freqtrade supports PostgreSQL by using SQLAlchemy, which supports multiple different database systems. + +Installation: +`pip install psycopg2` + +Usage: +`... --db-url postgresql+psycopg2://:@localhost:5432/` + +Freqtrade will automatically create the tables necessary upon startup. + +If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections. + +### MariaDB / MySQL + +Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems. + +Installation: +`pip install pymysql` + +Usage: +`... --db-url mysql+pymysql://:@localhost:3306/` diff --git a/docs/stoploss.md b/docs/stoploss.md index 671e643b0..ae191f639 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -51,6 +51,14 @@ The bot cannot do these every 5 seconds (at each iteration), otherwise it would So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute). This same logic will reapply a stoploss order on the exchange should you cancel it accidentally. +### forcesell + +`forcesell` is an optional value, which defaults to the same value as `sell` and is used when sending a `/forcesell` command from Telegram or from the Rest API. + +### forcebuy + +`forcebuy` is an optional value, which defaults to the same value as `buy` and is used when sending a `/forcebuy` command from Telegram or from the Rest API. + ### emergencysell `emergencysell` is an optional value, which defaults to `market` and is used when creating stop loss on exchange orders fails. diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index c051e2232..3436604a9 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -11,14 +11,118 @@ If you're just getting started, please be familiar with the methods described in !!! Tip You can get a strategy template containing all below methods by running `freqtrade new-strategy --strategy MyAwesomeStrategy --template advanced` +## Storing information + +Storing information can be accomplished by creating a new dictionary within the strategy class. + +The name of the variable can be chosen at will, but should be prefixed with `cust_` to avoid naming collisions with predefined strategy variables. + +```python +class AwesomeStrategy(IStrategy): + # Create custom dictionary + custom_info = {} + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # Check if the entry already exists + if not metadata["pair"] in self.custom_info: + # Create empty entry for this pair + self.custom_info[metadata["pair"]] = {} + + if "crosstime" in self.custom_info[metadata["pair"]]: + self.custom_info[metadata["pair"]]["crosstime"] += 1 + else: + self.custom_info[metadata["pair"]]["crosstime"] = 1 +``` + +!!! Warning + The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. + +!!! Note + If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. + +## Dataframe access + +You may access dataframe in various strategy functions by querying it from dataprovider. + +``` python +from freqtrade.exchange import timeframe_to_prev_date + +class AwesomeStrategy(IStrategy): + def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float, + rate: float, time_in_force: str, sell_reason: str, + current_time: 'datetime', **kwargs) -> bool: + # Obtain pair dataframe. + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) + + # Obtain last available candle. Do not use current_time to look up latest candle, because + # current_time points to curret incomplete candle whose data is not available. + last_candle = dataframe.iloc[-1].squeeze() + # <...> + + # In dry/live runs trade open date will not match candle open date therefore it must be + # rounded. + trade_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc) + # Look up trade candle. + trade_candle = dataframe.loc[dataframe['date'] == trade_date] + # trade_candle may be empty for trades that just opened as it is still incomplete. + if not trade_candle.empty: + trade_candle = trade_candle.squeeze() + # <...> +``` + +!!! Warning "Using .iloc[-1]" + You can use `.iloc[-1]` here because `get_analyzed_dataframe()` only returns candles that backtesting is allowed to see. + This will not work in `populate_*` methods, so make sure to not use `.iloc[]` in that area. + Also, this will only work starting with version 2021.5. + +*** + +## Custom sell signal + +It is possible to define custom sell signals, indicating that specified position should be sold. This is very useful when we need to customize sell conditions for each individual trade, or if you need the trade profit to take the sell decision. + +For example you could implement a 1:2 risk-reward ROI with `custom_sell()`. + +Using custom_sell() signals in place of stoplosses though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange. + +!!! Note + Returning a `string` or `True` from this method is equal to setting sell signal on a candle at specified time. This method is not called when sell signal is set already, or if sell signals are disabled (`use_sell_signal=False` or `sell_profit_only=True` while profit is below `sell_profit_offset`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters. + +An example of how we can use different indicators depending on the current profit and also sell trades that were open longer than one day: + +``` python +class AwesomeStrategy(IStrategy): + def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, + current_profit: float, **kwargs): + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) + last_candle = dataframe.iloc[-1].squeeze() + + # Above 20% profit, sell when rsi < 80 + if current_profit > 0.2: + if last_candle['rsi'] < 80: + return 'rsi_below_80' + + # Between 2% and 10%, sell if EMA-long above EMA-short + if 0.02 < current_profit < 0.1: + if last_candle['emalong'] > last_candle['emashort']: + return 'ema_long_below_80' + + # Sell any positions at a loss if they are held for more than one day. + if current_profit < 0.0 and (current_time - trade.open_date_utc).days >= 1: + return 'unclog' +``` + +See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks. + ## Custom stoploss -A stoploss can only ever move upwards - so if you set it to an absolute profit of 2%, you can never move it below this price. -Also, the traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss. +The stoploss price can only ever move upwards - if the stoploss value returned from `custom_stoploss` would result in a lower stoploss price than was previously set, it will be ignored. The traditional `stoploss` value serves as an absolute lower level and will be instated as the initial stoploss. The usage of the custom stoploss method must be enabled by setting `use_custom_stoploss=True` on the strategy object. -The method must return a stoploss value (float / number) with a relative ratio below the current price. -E.g. `current_profit = 0.05` (5% profit) - stoploss returns `0.02` - then you "locked in" a profit of 3% (`0.05 - 0.02 = 0.03`). +The method must return a stoploss value (float / number) as a percentage of the current price. +E.g. If the `current_rate` is 200 USD, then returning `0.02` will set the stoploss price 2% lower, at 196 USD. + +The absolute value of the return value is used (the sign is ignored), so returning `0.05` or `-0.05` have the same result, a stoploss 5% below the current price. To simulate a regular trailing stoploss of 4% (trailing 4% behind the maximum reached price) you would use the following very simple method: @@ -51,7 +155,7 @@ class AwesomeStrategy(IStrategy): :param current_rate: Rate, calculated based on pricing settings in ask_strategy. :param current_profit: Current profit (as ratio), calculated based on current_rate. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: New stoploss value, relative to the currentrate + :return float: New stoploss value, relative to the current rate """ return -0.04 ``` @@ -87,9 +191,9 @@ class AwesomeStrategy(IStrategy): current_rate: float, current_profit: float, **kwargs) -> float: # Make sure you have the longest interval first - these conditions are evaluated from top to bottom. - if current_time - timedelta(minutes=120) > trade.open_date: + if current_time - timedelta(minutes=120) > trade.open_date_utc: return -0.05 - elif current_time - timedelta(minutes=60) > trade.open_date: + elif current_time - timedelta(minutes=60) > trade.open_date_utc: return -0.10 return 1 ``` @@ -142,24 +246,31 @@ class AwesomeStrategy(IStrategy): return -1 # return a value bigger than the inital stoploss to keep using the inital stoploss # After reaching the desired offset, allow the stoploss to trail by half the profit - desired_stoploss = current_profit / 2 + desired_stoploss = current_profit / 2 # Use a minimum of 2.5% and a maximum of 5% return max(min(desired_stoploss, 0.05), 0.025) ``` -#### Absolute stoploss +#### Calculating stoploss relative to open price -The below example sets absolute profit levels based on the current profit. +Stoploss values returned from `custom_stoploss()` always specify a percentage relative to `current_rate`. In order to set a stoploss relative to the *open* price, we need to use `current_profit` to calculate what percentage relative to the `current_rate` will give you the same result as if the percentage was specified from the open price. + +The helper function [`stoploss_from_open()`](strategy-customization.md#stoploss_from_open) can be used to convert from an open price relative stop, to a current price relative stop which can be returned from `custom_stoploss()`. + +#### Stepped stoploss + +Instead of continuously trailing behind the current price, this example sets fixed stoploss price levels based on the current profit. * Use the regular stoploss until 20% profit is reached -* Once profit is > 40%, stoploss will be at 25%, locking in at least 25% of the profit. -* Once profit is > 25% - stoploss will be 15%. -* Once profit is > 20% - stoploss will be set to 7%. +* Once profit is > 20% - set stoploss to 7% above open price. +* Once profit is > 25% - set stoploss to 15% above open price. +* Once profit is > 40% - set stoploss to 25% above open price. ``` python from datetime import datetime from freqtrade.persistence import Trade +from freqtrade.strategy import stoploss_from_open class AwesomeStrategy(IStrategy): @@ -170,23 +281,57 @@ class AwesomeStrategy(IStrategy): def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, current_rate: float, current_profit: float, **kwargs) -> float: - # Calculate as `-desired_stop_from_open + current_profit` to get the distance between current_profit and initial price + # evaluate highest to lowest, so that highest possible stop is used if current_profit > 0.40: - return (-0.25 + current_profit) - if current_profit > 0.25: - return (-0.15 + current_profit) - if current_profit > 0.20: - return (-0.7 + current_profit) + return stoploss_from_open(0.25, current_profit) + elif current_profit > 0.25: + return stoploss_from_open(0.15, current_profit) + elif current_profit > 0.20: + return stoploss_from_open(0.07, current_profit) + + # return maximum stoploss value, keeping current stoploss price unchanged return 1 ``` +#### Custom stoploss using an indicator from dataframe example + +Absolute stoploss value may be derived from indicators stored in dataframe. Example uses parabolic SAR below the price as stoploss. + +``` python +class AwesomeStrategy(IStrategy): + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + # <...> + dataframe['sar'] = ta.SAR(dataframe) + + use_custom_stoploss = True + + def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, + current_rate: float, current_profit: float, **kwargs) -> float: + + dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) + last_candle = dataframe.iloc[-1].squeeze() + + # Use parabolic sar as absolute stoploss price + stoploss_price = last_candle['sar'] + + # Convert absolute price to percentage relative to current_rate + if stoploss_price < current_rate: + return (stoploss_price / current_rate) - 1 + + # return maximum stoploss value, keeping current stoploss price unchanged + return 1 +``` + +See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks. + --- ## Custom order timeout rules Simple, time-based order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section. -However, freqtrade also offers a custom callback for both order types, which allows you to decide based on custom criteria if a order did time out or not. +However, freqtrade also offers a custom callback for both order types, which allows you to decide based on custom criteria if an order did time out or not. !!! Note Unfilled order timeouts are not relevant during backtesting or hyperopt, and are only relevant during real (live) trading. Therefore these methods are only called in these circumstances. @@ -199,7 +344,7 @@ It applies a tight timeout for higher priced assets, while allowing more time to The function must return either `True` (cancel order) or `False` (keep order alive). ``` python -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from freqtrade.persistence import Trade class AwesomeStrategy(IStrategy): @@ -213,21 +358,21 @@ class AwesomeStrategy(IStrategy): } def check_buy_timeout(self, pair: str, trade: 'Trade', order: dict, **kwargs) -> bool: - if trade.open_rate > 100 and trade.open_date < datetime.utcnow() - timedelta(minutes=5): + if trade.open_rate > 100 and trade.open_date_utc < datetime.now(timezone.utc) - timedelta(minutes=5): return True - elif trade.open_rate > 10 and trade.open_date < datetime.utcnow() - timedelta(minutes=3): + elif trade.open_rate > 10 and trade.open_date_utc < datetime.now(timezone.utc) - timedelta(minutes=3): return True - elif trade.open_rate < 1 and trade.open_date < datetime.utcnow() - timedelta(hours=24): + elif trade.open_rate < 1 and trade.open_date_utc < datetime.now(timezone.utc) - timedelta(hours=24): return True return False def check_sell_timeout(self, pair: str, trade: 'Trade', order: dict, **kwargs) -> bool: - if trade.open_rate > 100 and trade.open_date < datetime.utcnow() - timedelta(minutes=5): + if trade.open_rate > 100 and trade.open_date_utc < datetime.now(timezone.utc) - timedelta(minutes=5): return True - elif trade.open_rate > 10 and trade.open_date < datetime.utcnow() - timedelta(minutes=3): + elif trade.open_rate > 10 and trade.open_date_utc < datetime.now(timezone.utc) - timedelta(minutes=3): return True - elif trade.open_rate < 1 and trade.open_date < datetime.utcnow() - timedelta(hours=24): + elif trade.open_rate < 1 and trade.open_date_utc < datetime.now(timezone.utc) - timedelta(hours=24): return True return False ``` @@ -412,7 +557,7 @@ Both attributes and methods may be overridden, altering behavior of the original ## Embedding Strategies -Freqtrade provides you with with an easy way to embed the strategy into your configuration file. +Freqtrade provides you with an easy way to embed the strategy into your configuration file. This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 7e998570f..cfea60d22 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -159,7 +159,7 @@ Edit the method `populate_buy_trend()` in your strategy file to update your buy It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected. -This will method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action". +This method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action". Sample from `user_data/strategies/sample_strategy.py`: @@ -193,7 +193,7 @@ Please note that the sell-signal is only used if `use_sell_signal` is set to tru It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected. -This will method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action". +This method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action". Sample from `user_data/strategies/sample_strategy.py`: @@ -300,38 +300,7 @@ The metadata-dict (available for `populate_buy_trend`, `populate_sell_trend`, `p Currently this is `pair`, which can be accessed using `metadata['pair']` - and will return a pair in the format `XRP/BTC`. The Metadata-dict should not be modified and does not persist information across multiple calls. -Instead, have a look at the section [Storing information](#Storing-information) - -### Storing information - -Storing information can be accomplished by creating a new dictionary within the strategy class. - -The name of the variable can be chosen at will, but should be prefixed with `cust_` to avoid naming collisions with predefined strategy variables. - -```python -class AwesomeStrategy(IStrategy): - # Create custom dictionary - cust_info = {} - - def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - # Check if the entry already exists - if not metadata["pair"] in self._cust_info: - # Create empty entry for this pair - self._cust_info[metadata["pair"]] = {} - - if "crosstime" in self.cust_info[metadata["pair"]: - self.cust_info[metadata["pair"]]["crosstime"] += 1 - else: - self.cust_info[metadata["pair"]]["crosstime"] = 1 -``` - -!!! Warning - The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash. - -!!! Note - If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. - -*** +Instead, have a look at the section [Storing information](strategy-advanced.md#Storing-information) ## Additional data (informative_pairs) @@ -399,7 +368,7 @@ if self.dp: ### *current_whitelist()* -Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. +Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. The strategy might look something like this: @@ -418,7 +387,7 @@ This is where calling `self.dp.current_whitelist()` comes in handy. pairs = self.dp.current_whitelist() # Assign tf to each pair so they can be downloaded and cached for strategy. informative_pairs = [(pair, '1d') for pair in pairs] - return informative_pairs + return informative_pairs ``` ### *get_pair_dataframe(pair, timeframe)* @@ -444,8 +413,9 @@ It can also be used in specific callbacks to get the signal that caused the acti ``` python # fetch current dataframe if self.dp: - dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'], - timeframe=self.timeframe) + if self.dp.runmode.value in ('live', 'dry_run'): + dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'], + timeframe=self.timeframe) ``` !!! Note "No data available" @@ -462,8 +432,28 @@ if self.dp: dataframe['best_ask'] = ob['asks'][0][0] ``` -!!! Warning - The order book is not part of the historic data which means backtesting and hyperopt will not work correctly if this method is used. +The orderbook structure is aligned with the order structure from [ccxt](https://github.com/ccxt/ccxt/wiki/Manual#order-book-structure), so the result will look as follows: + +``` js +{ + 'bids': [ + [ price, amount ], // [ float, float ] + [ price, amount ], + ... + ], + 'asks': [ + [ price, amount ], + [ price, amount ], + //... + ], + //... +} +``` + +Therefore, using `ob['bids'][0][0]` as demonstrated above will result in using the best bid price. `ob['bids'][0][1]` would look at the amount at this orderbook position. + +!!! Warning "Warning about backtesting" + The order book is not part of the historic data which means backtesting and hyperopt will not work correctly if this method is used, as the method will return uptodate values. ### *ticker(pair)* @@ -578,7 +568,7 @@ All columns of the informative dataframe will be available on the returning data ``` python 'date', 'open', 'high', 'low', 'close', 'rsi' # from the original dataframe - 'date_1h', 'open_1h', 'high_1h', 'low_1h', 'close_1h', 'rsi_1h' # from the informative dataframe + 'date_1h', 'open_1h', 'high_1h', 'low_1h', 'close_1h', 'rsi_1h' # from the informative dataframe ``` ??? Example "Custom implementation" @@ -613,6 +603,43 @@ All columns of the informative dataframe will be available on the returning data *** +### *stoploss_from_open()* + +Stoploss values returned from `custom_stoploss` must specify a percentage relative to `current_rate`, but sometimes you may want to specify a stoploss relative to the open price instead. `stoploss_from_open()` is a helper function to calculate a stoploss value that can be returned from `custom_stoploss` which will be equivalent to the desired percentage above the open price. + +??? Example "Returning a stoploss relative to the open price from the custom stoploss function" + + Say the open price was $100, and `current_price` is $121 (`current_profit` will be `0.21`). + + If we want a stop price at 7% above the open price we can call `stoploss_from_open(0.07, current_profit)` which will return `0.1157024793`. 11.57% below $121 is $107, which is the same as 7% above $100. + + + ``` python + + from datetime import datetime + from freqtrade.persistence import Trade + from freqtrade.strategy import IStrategy, stoploss_from_open + + class AwesomeStrategy(IStrategy): + + # ... populate_* methods + + use_custom_stoploss = True + + def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime, + current_rate: float, current_profit: float, **kwargs) -> float: + + # once the profit has risen above 10%, keep the stoploss at 7% above the open price + if current_profit > 0.10: + return stoploss_from_open(0.07, current_profit) + + return 1 + + ``` + + Full examples can be found in the [Custom stoploss](strategy-advanced.md#custom-stoploss) section of the Documentation. + + ## Additional data (Wallets) The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. @@ -704,7 +731,7 @@ To verify if a pair is currently locked, use `self.is_pair_locked(pair)`. Locked pairs will always be rounded up to the next candle. So assuming a `5m` timeframe, a lock with `until` set to 10:18 will lock the pair until the candle from 10:15-10:20 will be finished. !!! Warning - Locking pairs is not available during backtesting. + Manually locking pairs is not available during backtesting, only locks via Protections are allowed. #### Pair locking example diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 90e39fd76..4c938500c 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -24,7 +24,7 @@ config["strategy"] = "SampleStrategy" # Location of the data data_location = Path(config['user_data_dir'], 'data', 'binance') # Pair to analyze - Only use one pair here -pair = "BTC_USDT" +pair = "BTC/USDT" ``` @@ -34,7 +34,9 @@ from freqtrade.data.history import load_pair_history candles = load_pair_history(datadir=data_location, timeframe=config["timeframe"], - pair=pair) + pair=pair, + data_format = "hdf5", + ) # Confirm success print("Loaded " + str(len(candles)) + f" rows of data for {pair} from {data_location}") @@ -193,4 +195,18 @@ graph.show(renderer="browser") ``` +## Plot average profit per trade as distribution graph + + +```python +import plotly.figure_factory as ff + +hist_data = [trades.profit_ratio] +group_labels = ['profit_ratio'] # name of the dataset + +fig = ff.create_distplot(hist_data, group_labels,bin_size=0.01) +fig.show() + +``` + Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 57f2e98bd..87ff38881 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -72,21 +72,40 @@ Example configuration showing the different settings: ``` json "telegram": { - "enabled": true, - "token": "your_telegram_token", - "chat_id": "your_telegram_chat_id", - "notification_settings": { - "status": "silent", - "warning": "on", - "startup": "off", - "buy": "silent", - "sell": "on", - "buy_cancel": "silent", - "sell_cancel": "on" - } - }, + "enabled": true, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id", + "notification_settings": { + "status": "silent", + "warning": "on", + "startup": "off", + "buy": "silent", + "sell": { + "roi": "silent", + "emergency_sell": "on", + "force_sell": "on", + "sell_signal": "silent", + "trailing_stop_loss": "on", + "stop_loss": "on", + "stoploss_on_exchange": "on", + "custom_sell": "silent" + }, + "buy_cancel": "silent", + "sell_cancel": "on", + "buy_fill": "off", + "sell_fill": "off" + }, + "balance_dust_level": 0.01 +}, ``` +`buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange. +`sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange. +`*_fill` notifications are off by default and must be explicitly enabled. + + +`balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown. + ## Create a custom keyboard (command shortcut buttons) Telegram allows us to create a custom keyboard with buttons for commands. @@ -143,7 +162,8 @@ official commands. You can ask at any moment for help with `/help`. | `/delete ` | Delete a specific trade from the Database. Tries to close open orders. Requires manual handling of this trade on the exchange. | `/count` | Displays number of trades used and available | `/locks` | Show currently locked pairs. -| `/profit` | Display a summary of your profit/loss from close trades and some stats about your performance +| `/unlock ` | Remove the lock for this pair (or for this lock id). +| `/profit []` | Display a summary of your profit/loss from close trades and some stats about your performance, over the last n days (all trades by default) | `/forcesell ` | Instantly sells the given trade (Ignoring `minimum_roi`). | `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`). | `/forcebuy [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) @@ -239,10 +259,14 @@ Return a summary of your profit/loss and performance. > **BITTREX:** Selling BTC/LTC with limit `0.01650000 (profit: ~-4.07%, -0.00008168)` -### /forcebuy +### /forcebuy [rate] > **BITTREX:** Buying ETH/BTC with limit `0.03400000` (`1.000000 ETH`, `225.290 USD`) +Omitting the pair will open a query asking for the pair to buy (based on the current whitelist). + +![Telegram force-buy screenshot](assets/telegram_forcebuy.png) + Note that for this to work, `forcebuy_enable` needs to be set to true. [More details](configuration.md#understand-forcebuy_enable) @@ -250,12 +274,12 @@ Note that for this to work, `forcebuy_enable` needs to be set to true. ### /performance Return the performance of each crypto-currency the bot has sold. -> Performance: -> 1. `RCN/BTC 57.77%` -> 2. `PAY/BTC 56.91%` -> 3. `VIB/BTC 47.07%` -> 4. `SALT/BTC 30.24%` -> 5. `STORJ/BTC 27.24%` +> Performance: +> 1. `RCN/BTC 0.003 BTC (57.77%) (1)` +> 2. `PAY/BTC 0.0012 BTC (56.91%) (1)` +> 3. `VIB/BTC 0.0011 BTC (47.07%) (1)` +> 4. `SALT/BTC 0.0010 BTC (30.24%) (1)` +> 5. `STORJ/BTC 0.0009 BTC (27.24%) (1)` > ... ### /balance diff --git a/docs/utils.md b/docs/utils.md index cf7d5f1d1..8ef12e1c9 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -253,18 +253,211 @@ optional arguments: * Example: see exchanges available for the bot: ``` $ freqtrade list-exchanges -Exchanges available for Freqtrade: _1btcxe, acx, allcoin, bequant, bibox, binance, binanceje, binanceus, bitbank, bitfinex, bitfinex2, bitkk, bitlish, bitmart, bittrex, bitz, bleutrade, btcalpha, btcmarkets, btcturk, buda, cex, cobinhood, coinbaseprime, coinbasepro, coinex, cointiger, coss, crex24, digifinex, dsx, dx, ethfinex, fcoin, fcoinjp, gateio, gdax, gemini, hitbtc2, huobipro, huobiru, idex, kkex, kraken, kucoin, kucoin2, kuna, lbank, mandala, mercado, oceanex, okcoincny, okcoinusd, okex, okex3, poloniex, rightbtc, theocean, tidebit, upbit, zb +Exchanges available for Freqtrade: +Exchange name Valid reason +--------------- ------- -------------------------------------------- +aax True +ascendex True missing opt: fetchMyTrades +bequant True +bibox True +bigone True +binance True +binanceus True +bitbank True missing opt: fetchTickers +bitcoincom True +bitfinex True +bitforex True missing opt: fetchMyTrades, fetchTickers +bitget True +bithumb True missing opt: fetchMyTrades +bitkk True missing opt: fetchMyTrades +bitmart True +bitmax True missing opt: fetchMyTrades +bitpanda True +bittrex True +bitvavo True +bitz True missing opt: fetchMyTrades +btcalpha True missing opt: fetchTicker, fetchTickers +btcmarkets True missing opt: fetchTickers +buda True missing opt: fetchMyTrades, fetchTickers +bw True missing opt: fetchMyTrades, fetchL2OrderBook +bybit True +bytetrade True +cdax True +cex True missing opt: fetchMyTrades +coinbaseprime True missing opt: fetchTickers +coinbasepro True missing opt: fetchTickers +coinex True +crex24 True +deribit True +digifinex True +equos True missing opt: fetchTicker, fetchTickers +eterbase True +fcoin True missing opt: fetchMyTrades, fetchTickers +fcoinjp True missing opt: fetchMyTrades, fetchTickers +ftx True +gateio True +gemini True +gopax True +hbtc True +hitbtc True +huobijp True +huobipro True +idex True +kraken True +kucoin True +lbank True missing opt: fetchMyTrades +mercado True missing opt: fetchTickers +ndax True missing opt: fetchTickers +novadax True +okcoin True +okex True +probit True +qtrade True +stex True +timex True +upbit True missing opt: fetchMyTrades +vcc True +zb True missing opt: fetchMyTrades + ``` +!!! Note "missing opt exchanges" + Values with "missing opt:" might need special configuration (e.g. using orderbook if `fetchTickers` is missing) - but should in theory work (although we cannot guarantee they will). + * Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade): ``` $ freqtrade list-exchanges -a -All exchanges supported by the ccxt library: _1btcxe, acx, adara, allcoin, anxpro, bcex, bequant, bibox, bigone, binance, binanceje, binanceus, bit2c, bitbank, bitbay, bitfinex, bitfinex2, bitflyer, bitforex, bithumb, bitkk, bitlish, bitmart, bitmex, bitso, bitstamp, bitstamp1, bittrex, bitz, bl3p, bleutrade, braziliex, btcalpha, btcbox, btcchina, btcmarkets, btctradeim, btctradeua, btcturk, buda, bxinth, cex, chilebit, cobinhood, coinbase, coinbaseprime, coinbasepro, coincheck, coinegg, coinex, coinexchange, coinfalcon, coinfloor, coingi, coinmarketcap, coinmate, coinone, coinspot, cointiger, coolcoin, coss, crex24, crypton, deribit, digifinex, dsx, dx, ethfinex, exmo, exx, fcoin, fcoinjp, flowbtc, foxbit, fybse, gateio, gdax, gemini, hitbtc, hitbtc2, huobipro, huobiru, ice3x, idex, independentreserve, indodax, itbit, kkex, kraken, kucoin, kucoin2, kuna, lakebtc, latoken, lbank, liquid, livecoin, luno, lykke, mandala, mercado, mixcoins, negociecoins, nova, oceanex, okcoincny, okcoinusd, okex, okex3, paymium, poloniex, rightbtc, southxchange, stronghold, surbitcoin, theocean, therock, tidebit, tidex, upbit, vaultoro, vbtc, virwox, xbtce, yobit, zaif, zb +All exchanges supported by the ccxt library: +Exchange name Valid reason +------------------ ------- --------------------------------------------------------------------------------------- +aax True +aofex False missing: fetchOrder +ascendex True missing opt: fetchMyTrades +bequant True +bibox True +bigone True +binance True +binanceus True +bit2c False missing: fetchOrder, fetchOHLCV +bitbank True missing opt: fetchTickers +bitbay False missing: fetchOrder +bitcoincom True +bitfinex True +bitfinex2 False missing: fetchOrder +bitflyer False missing: fetchOrder, fetchOHLCV +bitforex True missing opt: fetchMyTrades, fetchTickers +bitget True +bithumb True missing opt: fetchMyTrades +bitkk True missing opt: fetchMyTrades +bitmart True +bitmax True missing opt: fetchMyTrades +bitmex False Various reasons. +bitpanda True +bitso False missing: fetchOHLCV +bitstamp False Does not provide history. Details in https://github.com/freqtrade/freqtrade/issues/1983 +bitstamp1 False missing: fetchOrder, fetchOHLCV +bittrex True +bitvavo True +bitz True missing opt: fetchMyTrades +bl3p False missing: fetchOrder, fetchOHLCV +bleutrade False missing: fetchOrder +braziliex False missing: fetchOHLCV +btcalpha True missing opt: fetchTicker, fetchTickers +btcbox False missing: fetchOHLCV +btcmarkets True missing opt: fetchTickers +btctradeua False missing: fetchOrder, fetchOHLCV +btcturk False missing: fetchOrder +buda True missing opt: fetchMyTrades, fetchTickers +bw True missing opt: fetchMyTrades, fetchL2OrderBook +bybit True +bytetrade True +cdax True +cex True missing opt: fetchMyTrades +chilebit False missing: fetchOrder, fetchOHLCV +coinbase False missing: fetchOrder, cancelOrder, createOrder, fetchOHLCV +coinbaseprime True missing opt: fetchTickers +coinbasepro True missing opt: fetchTickers +coincheck False missing: fetchOrder, fetchOHLCV +coinegg False missing: fetchOHLCV +coinex True +coinfalcon False missing: fetchOHLCV +coinfloor False missing: fetchOrder, fetchOHLCV +coingi False missing: fetchOrder, fetchOHLCV +coinmarketcap False missing: fetchOrder, cancelOrder, createOrder, fetchBalance, fetchOHLCV +coinmate False missing: fetchOHLCV +coinone False missing: fetchOHLCV +coinspot False missing: fetchOrder, cancelOrder, fetchOHLCV +crex24 True +currencycom False missing: fetchOrder +delta False missing: fetchOrder +deribit True +digifinex True +equos True missing opt: fetchTicker, fetchTickers +eterbase True +exmo False missing: fetchOrder +exx False missing: fetchOHLCV +fcoin True missing opt: fetchMyTrades, fetchTickers +fcoinjp True missing opt: fetchMyTrades, fetchTickers +flowbtc False missing: fetchOrder, fetchOHLCV +foxbit False missing: fetchOrder, fetchOHLCV +ftx True +gateio True +gemini True +gopax True +hbtc True +hitbtc True +hollaex False missing: fetchOrder +huobijp True +huobipro True +idex True +independentreserve False missing: fetchOHLCV +indodax False missing: fetchOHLCV +itbit False missing: fetchOHLCV +kraken True +kucoin True +kuna False missing: fetchOHLCV +lakebtc False missing: fetchOrder, fetchOHLCV +latoken False missing: fetchOrder, fetchOHLCV +lbank True missing opt: fetchMyTrades +liquid False missing: fetchOHLCV +luno False missing: fetchOHLCV +lykke False missing: fetchOHLCV +mercado True missing opt: fetchTickers +mixcoins False missing: fetchOrder, fetchOHLCV +ndax True missing opt: fetchTickers +novadax True +oceanex False missing: fetchOHLCV +okcoin True +okex True +paymium False missing: fetchOrder, fetchOHLCV +phemex False Does not provide history. +poloniex False missing: fetchOrder +probit True +qtrade True +rightbtc False missing: fetchOrder +ripio False missing: fetchOHLCV +southxchange False missing: fetchOrder, fetchOHLCV +stex True +surbitcoin False missing: fetchOrder, fetchOHLCV +therock False missing: fetchOHLCV +tidebit False missing: fetchOrder +tidex False missing: fetchOHLCV +timex True +upbit True missing opt: fetchMyTrades +vbtc False missing: fetchOrder, fetchOHLCV +vcc True +wavesexchange False missing: fetchOrder +whitebit False missing: fetchOrder, cancelOrder, createOrder, fetchBalance +xbtce False missing: fetchOrder, fetchOHLCV +xena False missing: fetchOrder +yobit False missing: fetchOHLCV +zaif False missing: fetchOrder, fetchOHLCV +zb True missing opt: fetchMyTrades ``` ## List Timeframes -Use the `list-timeframes` subcommand to see the list of timeframes (ticker intervals) available for the exchange. +Use the `list-timeframes` subcommand to see the list of timeframes available for the exchange. ``` usage: freqtrade list-timeframes [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [-1] diff --git a/docs/webhook-config.md b/docs/webhook-config.md index db6d4d1ef..8ce6edc18 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -19,6 +19,11 @@ Sample configuration (tested using IFTTT). "value1": "Cancelling Open Buy Order for {pair}", "value2": "limit {limit:8f}", "value3": "{stake_amount:8f} {stake_currency}" + }, + "webhookbuyfill": { + "value1": "Buy Order for {pair} filled", + "value2": "at {open_rate:8f}", + "value3": "" }, "webhooksell": { "value1": "Selling {pair}", @@ -30,6 +35,11 @@ Sample configuration (tested using IFTTT). "value2": "limit {limit:8f}", "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})" }, + "webhooksellfill": { + "value1": "Sell Order for {pair} filled", + "value2": "at {close_rate:8f}.", + "value3": "" + }, "webhookstatus": { "value1": "Status: {status}", "value2": "", @@ -40,6 +50,21 @@ Sample configuration (tested using IFTTT). The url in `webhook.url` should point to the correct url for your webhook. If you're using [IFTTT](https://ifttt.com) (as shown in the sample above) please insert our event and key to the url. +You can set the POST body format to Form-Encoded (default) or JSON-Encoded. Use `"format": "form"` or `"format": "json"` respectively. Example configuration for Mattermost Cloud integration: + +```json + "webhook": { + "enabled": true, + "url": "https://.cloud.mattermost.com/hooks/", + "format": "json", + "webhookstatus": { + "text": "Status: {status}" + } + }, +``` + +The result would be POST request with e.g. `{"text":"Status: running"}` body and `Content-Type: application/json` header which results `Status: running` message in the Mattermost channel. + Different payloads can be configured for different events. Not all fields are necessary, but you should configure at least one of the dicts, otherwise the webhook will never be called. ### Webhookbuy @@ -76,6 +101,21 @@ Possible parameters are: * `order_type` * `current_rate` +### Webhookbuyfill + +The fields in `webhook.webhookbuyfill` are filled when the bot filled a buy order. Parameters are filled using string.format. +Possible parameters are: + +* `trade_id` +* `exchange` +* `pair` +* `open_rate` +* `amount` +* `open_date` +* `stake_amount` +* `stake_currency` +* `fiat_currency` + ### Webhooksell The fields in `webhook.webhooksell` are filled when the bot sells a trade. Parameters are filled using string.format. @@ -88,6 +128,27 @@ Possible parameters are: * `limit` * `amount` * `open_rate` +* `profit_amount` +* `profit_ratio` +* `stake_currency` +* `fiat_currency` +* `sell_reason` +* `order_type` +* `open_date` +* `close_date` + +### Webhooksellfill + +The fields in `webhook.webhooksellfill` are filled when the bot fills a sell order (closes a Trae). Parameters are filled using string.format. +Possible parameters are: + +* `trade_id` +* `exchange` +* `pair` +* `gain` +* `close_rate` +* `amount` +* `open_rate` * `current_rate` * `profit_amount` * `profit_ratio` diff --git a/docs/windows_installation.md b/docs/windows_installation.md index 168938973..edc0a1404 100644 --- a/docs/windows_installation.md +++ b/docs/windows_installation.md @@ -1,3 +1,5 @@ +# Windows installation + We **strongly** recommend that Windows users use [Docker](docker_quickstart.md) as this will work much easier and smoother (also more secure). If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work. @@ -21,7 +23,7 @@ git clone https://github.com/freqtrade/freqtrade.git Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.19‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version) +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.20‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version). Freqtrade provides these dependencies for the latest 2 Python versions (3.7 and 3.8) and for 64bit Windows. Other versions must be downloaded from the above link. diff --git a/environment.yml b/environment.yml index 938b5b6b8..f58434c15 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: # - defaults dependencies: # 1/4 req main - - python>=3.7 + - python>=3.7,<3.9 - numpy - pandas - pip diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index c64c11a18..7f4f7edd6 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -14,18 +14,18 @@ ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_dat ARGS_STRATEGY = ["strategy", "strategy_path"] -ARGS_TRADE = ["db_url", "sd_notify", "dry_run"] +ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"] ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", - "max_open_trades", "stake_amount", "fee"] + "max_open_trades", "stake_amount", "fee", "pairs"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", - "enable_protections", + "enable_protections", "dry_run_wallet", "strategy_list", "export", "exportfilename"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "position_stacking", "use_max_market_positions", - "enable_protections", + "enable_protections", "dry_run_wallet", "epochs", "spaces", "print_all", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", @@ -60,15 +60,16 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] -ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange", - "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] +ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange", + "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv", + "dataformat_trades"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", "timerange", "timeframe", "no_trades"] ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", - "trade_source", "timeframe"] + "trade_source", "timeframe", "plot_auto_open"] ARGS_INSTALL_UI = ["erase_ui_only"] diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 7bdbcc057..03d095e12 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -1,9 +1,11 @@ import logging +import secrets from pathlib import Path from typing import Any, Dict, List from questionary import Separator, prompt +from freqtrade.configuration.directory_operations import chown_user_directory from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.exceptions import OperationalException from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges @@ -93,10 +95,10 @@ def ask_user_config() -> Dict[str, Any]: "message": "Select exchange", "choices": [ "binance", - "binanceje", "binanceus", "bittrex", "kraken", + "ftx", Separator(), "other", ], @@ -138,6 +140,32 @@ def ask_user_config() -> Dict[str, Any]: "message": "Insert Telegram chat id", "when": lambda x: x['telegram'] }, + { + "type": "confirm", + "name": "api_server", + "message": "Do you want to enable the Rest API (includes FreqUI)?", + "default": False, + }, + { + "type": "text", + "name": "api_server_listen_addr", + "message": "Insert Api server Listen Address (best left untouched default!)", + "default": "127.0.0.1", + "when": lambda x: x['api_server'] + }, + { + "type": "text", + "name": "api_server_username", + "message": "Insert api-server username", + "default": "freqtrader", + "when": lambda x: x['api_server'] + }, + { + "type": "text", + "name": "api_server_password", + "message": "Insert api-server password", + "when": lambda x: x['api_server'] + }, ] answers = prompt(questions) @@ -145,6 +173,9 @@ def ask_user_config() -> Dict[str, Any]: # Interrupted questionary sessions return an empty dict. raise OperationalException("User interrupted interactive questions.") + # Force JWT token to be a random string + answers['api_server_jwt_key'] = secrets.token_hex() + return answers @@ -173,6 +204,9 @@ def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: arguments=selections) logger.info(f"Writing config to `{config_path}`.") + logger.info( + "Please make sure to check the configuration contents and adjust settings to your needs.") + config_path.write_text(config_text) @@ -183,6 +217,7 @@ def start_new_config(args: Dict[str, Any]) -> None: """ config_path = Path(args['config'][0]) + chown_user_directory(config_path.parent) if config_path.exists(): overwrite = ask_user_overwrite(config_path) if overwrite: diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 7dc85377d..d832693ee 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -110,10 +110,15 @@ AVAILABLE_CLI_OPTIONS = { help='Enforce dry-run for trading (removes Exchange secrets and simulates trades).', action='store_true', ), + "dry_run_wallet": Arg( + '--dry-run-wallet', '--starting-balance', + help='Starting balance, used for backtesting / hyperopt and dry-runs.', + type=float, + ), # Optimize common "timeframe": Arg( '-i', '--timeframe', '--ticker-interval', - help='Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`).', + help='Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).', ), "timerange": Arg( '--timerange', @@ -128,7 +133,6 @@ AVAILABLE_CLI_OPTIONS = { "stake_amount": Arg( '--stake-amount', help='Override the value of the `stake_amount` configuration setting.', - type=float, ), # Backtesting "position_stacking": Arg( @@ -191,6 +195,7 @@ AVAILABLE_CLI_OPTIONS = { '--hyperopt', help='Specify hyperopt class name which will be used by the bot.', metavar='NAME', + required=False, ), "hyperopt_path": Arg( '--hyperopt-path', @@ -262,7 +267,7 @@ AVAILABLE_CLI_OPTIONS = { default=1, ), "hyperopt_loss": Arg( - '--hyperopt-loss', + '--hyperopt-loss', '--hyperoptloss', help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). ' 'Different functions can generate completely different results, ' 'since the target for optimization is different. Built-in Hyperopt-loss-functions are: ' @@ -325,7 +330,7 @@ AVAILABLE_CLI_OPTIONS = { # Script options "pairs": Arg( '-p', '--pairs', - help='Show profits for only these pairs. Pairs are space-separated.', + help='Limit command to these pairs. Pairs are space-separated.', nargs='+', ), # Download data @@ -340,6 +345,12 @@ AVAILABLE_CLI_OPTIONS = { type=check_int_positive, metavar='INT', ), + "new_pairs_days": Arg( + '--new-pairs-days', + help='Download data of new pairs for given number of days. Default: `%(default)s`.', + type=check_int_positive, + metavar='INT', + ), "download_trades": Arg( '--dl-trades', help='Download trades instead of OHLCV data. The bot will resample trades to the ' @@ -422,6 +433,11 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=750, ), + "plot_auto_open": Arg( + '--auto-open', + help='Automatically open generated plot.', + action='store_true', + ), "no_trades": Arg( '--no-trades', help='Skip using trades from backtesting file and DB.', diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 1ce02eee5..3877e0801 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -8,11 +8,11 @@ from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -62,8 +62,8 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], - timerange=timerange, erase=bool(config.get('erase')), - data_format=config['dataformat_trades']) + timerange=timerange, new_pairs_days=config['new_pairs_days'], + erase=bool(config.get('erase')), data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( @@ -75,8 +75,9 @@ def start_download_data(args: Dict[str, Any]) -> None: else: pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=expanded_pairs, timeframes=config['timeframes'], - datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), - data_format=config['dataformat_ohlcv']) + datadir=config['datadir'], timerange=timerange, + new_pairs_days=config['new_pairs_days'], + erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 5ba3db9f9..cc0d653b9 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -8,9 +8,9 @@ import requests from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template, render_template_with_fallback -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index fd8f737f0..19337b407 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -6,8 +6,9 @@ from colorama import init as colorama_init from freqtrade.configuration import setup_utils_configuration from freqtrade.data.btanalysis import get_latest_hyperopt_file +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode +from freqtrade.optimize.optimize_reports import show_backtest_result logger = logging.getLogger(__name__) @@ -17,7 +18,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: """ List hyperopt epochs previously evaluated """ - from freqtrade.optimize.hyperopt import Hyperopt + from freqtrade.optimize.hyperopt_tools import HyperoptTools config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -47,7 +48,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: config.get('hyperoptexportfilename')) # Previous evaluations - epochs = Hyperopt.load_previous_results(results_file) + epochs = HyperoptTools.load_previous_results(results_file) total_epochs = len(epochs) epochs = hyperopt_filter_epochs(epochs, filteroptions) @@ -57,18 +58,19 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if not export_csv: try: - print(Hyperopt.get_result_table(config, epochs, total_epochs, - not filteroptions['only_best'], print_colorized, 0)) + print(HyperoptTools.get_result_table(config, epochs, total_epochs, + not filteroptions['only_best'], + print_colorized, 0)) except KeyboardInterrupt: print('User interrupted..') if epochs and not no_details: sorted_epochs = sorted(epochs, key=itemgetter('loss')) results = sorted_epochs[0] - Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) + HyperoptTools.show_epoch_details(results, total_epochs, print_json, no_header) if epochs and export_csv: - Hyperopt.export_csv_file( + HyperoptTools.export_csv_file( config, epochs, total_epochs, not filteroptions['only_best'], export_csv ) @@ -77,7 +79,7 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: """ Show details of a hyperopt epoch previously evaluated """ - from freqtrade.optimize.hyperopt import Hyperopt + from freqtrade.optimize.hyperopt_tools import HyperoptTools config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) @@ -105,7 +107,7 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: } # Previous evaluations - epochs = Hyperopt.load_previous_results(results_file) + epochs = HyperoptTools.load_previous_results(results_file) total_epochs = len(epochs) epochs = hyperopt_filter_epochs(epochs, filteroptions) @@ -124,18 +126,26 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: if epochs: val = epochs[n] - Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, - header_str="Epoch details") + + metrics = val['results_metrics'] + if 'strategy_name' in metrics: + show_backtest_result(metrics['strategy_name'], metrics, + metrics['stake_currency']) + + HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, + header_str="Epoch details") def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: """ Filter our items from the list of hyperopt results + TODO: after 2021.5 remove all "legacy" mode queries. """ if filteroptions['only_best']: epochs = [x for x in epochs if x['is_best']] if filteroptions['only_profitable']: - epochs = [x for x in epochs if x['results_metrics']['profit'] > 0] + epochs = [x for x in epochs if x['results_metrics'].get( + 'profit', x['results_metrics'].get('profit_total', 0)) > 0] epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions) @@ -152,34 +162,59 @@ def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: return epochs +def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int): + """ + Filter epochs with trade-counts > trades + """ + return [ + x for x in epochs + if x['results_metrics'].get( + 'trade_count', x['results_metrics'].get('total_trades', 0) + ) > trade_count + ] + + def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List: if filteroptions['filter_min_trades'] > 0: - epochs = [ - x for x in epochs - if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] - ] + epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades']) + if filteroptions['filter_max_trades'] > 0: epochs = [ x for x in epochs - if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] + if x['results_metrics'].get( + 'trade_count', x['results_metrics'].get('total_trades') + ) < filteroptions['filter_max_trades'] ] return epochs def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List: + def get_duration_value(x): + # Duration in minutes ... + if 'duration' in x['results_metrics']: + return x['results_metrics']['duration'] + else: + # New mode + if 'holding_avg_s' in x['results_metrics']: + avg = x['results_metrics']['holding_avg_s'] + return avg // 60 + raise OperationalException( + "Holding-average not available. Please omit the filter on average time, " + "or rerun hyperopt with this version") + if filteroptions['filter_min_avg_time'] is not None: - epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] + if get_duration_value(x) > filteroptions['filter_min_avg_time'] ] if filteroptions['filter_max_avg_time'] is not None: - epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] + if get_duration_value(x) < filteroptions['filter_max_avg_time'] ] return epochs @@ -188,28 +223,36 @@ def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List: def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: if filteroptions['filter_min_avg_profit'] is not None: - epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit'] + if x['results_metrics'].get( + 'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100 + ) > filteroptions['filter_min_avg_profit'] ] if filteroptions['filter_max_avg_profit'] is not None: - epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics']['avg_profit'] < filteroptions['filter_max_avg_profit'] + if x['results_metrics'].get( + 'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100 + ) < filteroptions['filter_max_avg_profit'] ] if filteroptions['filter_min_total_profit'] is not None: - epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] + if x['results_metrics'].get( + 'profit', x['results_metrics'].get('profit_total_abs', 0) + ) > filteroptions['filter_min_total_profit'] ] if filteroptions['filter_max_total_profit'] is not None: - epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [ x for x in epochs - if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] + if x['results_metrics'].get( + 'profit', x['results_metrics'].get('profit_total_abs', 0) + ) < filteroptions['filter_max_total_profit'] ] return epochs @@ -217,11 +260,11 @@ def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List: if filteroptions['filter_min_objective'] is not None: - epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']] if filteroptions['filter_max_objective'] is not None: - epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = _hyperopt_filter_epochs_trade(epochs, 0) epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']] diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 9e6076dfb..cd26aa60e 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -1,7 +1,6 @@ import csv import logging import sys -from collections import OrderedDict from pathlib import Path from typing import Any, Dict, List @@ -12,11 +11,11 @@ from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.exchange import available_exchanges, ccxt_exchanges, market_is_active +from freqtrade.exchange import market_is_active, validate_exchanges from freqtrade.misc import plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -28,14 +27,18 @@ def start_list_exchanges(args: Dict[str, Any]) -> None: :param args: Cli args from Arguments() :return: None """ - exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges() + exchanges = validate_exchanges(args['list_exchanges_all']) + if args['print_one_column']: - print('\n'.join(exchanges)) + print('\n'.join([e[0] for e in exchanges])) else: if args['list_exchanges_all']: - print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}") + print("All exchanges supported by the ccxt library:") else: - print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") + print("Exchanges available for Freqtrade:") + exchanges = [e for e in exchanges if e[1] is not False] + + print(tabulate(exchanges, headers=['Exchange name', 'Valid', 'reason'])) def _print_objs_tabular(objs: List, print_colorized: bool) -> None: @@ -50,15 +53,21 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: reset = '' names = [s['name'] for s in objs] - objss_to_print = [{ + objs_to_print = [{ 'name': s['name'] if s['name'] else "--", 'location': s['location'].name, 'status': (red + "LOAD FAILED" + reset if s['class'] is None else "OK" if names.count(s['name']) == 1 else yellow + "DUPLICATE NAME" + reset) } for s in objs] - - print(tabulate(objss_to_print, headers='keys', tablefmt='psql', stralign='right')) + for idx, s in enumerate(objs): + if 'hyperoptable' in s: + objs_to_print[idx].update({ + 'hyperoptable': "Yes" if s['hyperoptable']['count'] > 0 else "No", + 'buy-Params': len(s['hyperoptable'].get('buy', [])), + 'sell-Params': len(s['hyperoptable'].get('sell', [])), + }) + print(tabulate(objs_to_print, headers='keys', tablefmt='psql', stralign='right')) def start_list_strategies(args: Dict[str, Any]) -> None: @@ -71,6 +80,11 @@ def start_list_strategies(args: Dict[str, Any]) -> None: strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column']) # Sort alphabetically strategy_objs = sorted(strategy_objs, key=lambda x: x['name']) + for obj in strategy_objs: + if obj['class']: + obj['hyperoptable'] = obj['class'].detect_all_parameters() + else: + obj['hyperoptable'] = {'count': 0} if args['print_one_column']: print('\n'.join([s['name'] for s in strategy_objs])) @@ -99,7 +113,7 @@ def start_list_hyperopts(args: Dict[str, Any]) -> None: def start_list_timeframes(args: Dict[str, Any]) -> None: """ - Print ticker intervals (timeframes) available on Exchange + Print timeframes available on Exchange """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) # Do not use timeframe set in the config @@ -139,7 +153,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: pairs_only=pairs_only, active_only=active_only) # Sort the pairs/markets by symbol - pairs = OrderedDict(sorted(pairs.items())) + pairs = dict(sorted(pairs.items())) except Exception as e: raise OperationalException(f"Cannot get markets. Reason: {e}") from e @@ -177,7 +191,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: # human-readable formats. print() - if len(pairs): + if pairs: if args.get('print_list', False): # print data as a list, with human-readable summary print(f"{summary_str}: {', '.join(pairs.keys())}.") diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 7411ca9c6..a84b3b3bd 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -3,8 +3,9 @@ from typing import Any, Dict from freqtrade import constants from freqtrade.configuration import setup_utils_configuration -from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.state import RunMode +from freqtrade.enums import RunMode +from freqtrade.exceptions import OperationalException +from freqtrade.misc import round_coin_value logger = logging.getLogger(__name__) @@ -22,11 +23,13 @@ def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[ RunMode.BACKTEST: 'backtesting', RunMode.HYPEROPT: 'hyperoptimization', } - if (method in no_unlimited_runmodes.keys() and - config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT): - raise DependencyException( - f'The value of `stake_amount` cannot be set as "{constants.UNLIMITED_STAKE_AMOUNT}" ' - f'for {no_unlimited_runmodes[method]}') + if method in no_unlimited_runmodes.keys(): + if (config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT + and config['stake_amount'] > config['dry_run_wallet']): + wallet = round_coin_value(config['dry_run_wallet'], config['stake_currency']) + stake = round_coin_value(config['stake_amount'], config['stake_currency']) + raise OperationalException(f"Starting balance ({wallet}) " + f"is smaller than stake_amount {stake}.") return config diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index 0661cd03c..9f7a5958e 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -4,8 +4,8 @@ from typing import Any, Dict import rapidjson from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.resolvers import ExchangeResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -31,7 +31,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: results[curr] = pairlists.whitelist for curr, pairlist in results.items(): - if not args.get('print_one_column', False): + if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False): print(f"Pairs for {curr}: ") if args.get('print_one_column', False): diff --git a/freqtrade/commands/plot_commands.py b/freqtrade/commands/plot_commands.py index 5e547acb0..73a429a28 100644 --- a/freqtrade/commands/plot_commands.py +++ b/freqtrade/commands/plot_commands.py @@ -1,8 +1,8 @@ from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode def validate_plot_args(args: Dict[str, Any]) -> None: diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index aa36de3ff..f282447d4 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -1,10 +1,10 @@ import logging from typing import Any, Dict +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, is_exchange_bad, - is_exchange_known_ccxt, is_exchange_officially_supported) -from freqtrade.state import RunMode +from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt, + is_exchange_officially_supported, validate_exchange) logger = logging.getLogger(__name__) @@ -57,9 +57,13 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: f'{", ".join(available_exchanges())}' ) - if check_for_bad and is_exchange_bad(exchange): - raise OperationalException(f'Exchange "{exchange}" is known to not work with the bot yet. ' - f'Reason: {get_exchange_bad_reason(exchange)}') + valid, reason = validate_exchange(exchange) + if not valid: + if check_for_bad: + raise OperationalException(f'Exchange "{exchange}" will not work with Freqtrade. ' + f'Reason: {reason}') + else: + logger.warning(f'Exchange "{exchange}" will not work with Freqtrade. Reason: {reason}') if is_exchange_officially_supported(exchange): logger.info(f'Exchange "{exchange}" is officially supported ' diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 3b0f778f4..cd8464ead 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -1,7 +1,7 @@ import logging from typing import Any, Dict -from freqtrade.state import RunMode +from freqtrade.enums import RunMode from .check_exchange import remove_credentials from .config_validation import validate_config_consistency diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 187b2e3c7..3004d6bf7 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -6,8 +6,8 @@ from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match from freqtrade import constants +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -47,6 +47,8 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: conf_schema = deepcopy(constants.CONF_SCHEMA) if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE): conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED + elif conf.get('runmode', RunMode.OTHER) in (RunMode.BACKTEST, RunMode.HYPEROPT): + conf_schema['required'] = constants.SCHEMA_BACKTEST_REQUIRED else: conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED try: @@ -72,6 +74,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: # validating trailing stoploss _validate_trailing_stoploss(conf) + _validate_price_config(conf) _validate_edge(conf) _validate_whitelist(conf) _validate_protections(conf) @@ -93,6 +96,19 @@ def _validate_unlimited_amount(conf: Dict[str, Any]) -> None: raise OperationalException("`max_open_trades` and `stake_amount` cannot both be unlimited.") +def _validate_price_config(conf: Dict[str, Any]) -> None: + """ + When using market orders, price sides must be using the "other" side of the price + """ + if (conf.get('order_types', {}).get('buy') == 'market' + and conf.get('bid_strategy', {}).get('price_side') != 'ask'): + raise OperationalException('Market buy orders require bid_strategy.price_side = "ask".') + + if (conf.get('order_types', {}).get('sell') == 'market' + and conf.get('ask_strategy', {}).get('price_side') != 'bid'): + raise OperationalException('Market sell orders require ask_strategy.price_side = "bid".') + + def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: if conf.get('stoploss') == 0.0: @@ -133,11 +149,6 @@ def _validate_edge(conf: Dict[str, Any]) -> None: if not conf.get('edge', {}).get('enabled'): return - if conf.get('pairlist', {}).get('method') == 'VolumePairList': - raise OperationalException( - "Edge and VolumePairList are incompatible, " - "Edge will override whatever pairs VolumePairlist selects." - ) if not conf.get('ask_strategy', {}).get('use_sell_signal', True): raise OperationalException( "Edge requires `use_sell_signal` to be True, otherwise no sells will happen." diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 7bf3e6bf2..bfeb2da5c 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -11,11 +11,11 @@ from freqtrade import constants from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir -from freqtrade.configuration.load_config import load_config_file +from freqtrade.configuration.load_config import load_config_file, load_file +from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging -from freqtrade.misc import deep_merge_dicts, json_load -from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode +from freqtrade.misc import deep_merge_dicts logger = logging.getLogger(__name__) @@ -75,8 +75,6 @@ class Configuration: # Normalize config if 'internals' not in config: config['internals'] = {} - # TODO: This can be deleted along with removal of deprecated - # experimental settings if 'ask_strategy' not in config: config['ask_strategy'] = {} @@ -108,6 +106,8 @@ class Configuration: self._process_plot_options(config) + self._process_data_options(config) + # Check if the exchange set by the user is supported check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True)) @@ -214,9 +214,6 @@ class Configuration: self._args_to_config( config, argname='enable_protections', logstring='Parameter --enable-protections detected, enabling Protections. ...') - # Setting max_open_trades to infinite if -1 - if config.get('max_open_trades') == -1: - config['max_open_trades'] = float('inf') if 'use_max_market_positions' in self.args and not self.args["use_max_market_positions"]: config.update({'use_max_market_positions': False}) @@ -228,11 +225,23 @@ class Configuration: 'overriding max_open_trades to: %s ...', config.get('max_open_trades')) elif config['runmode'] in NON_UTIL_MODES: logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) + # Setting max_open_trades to infinite if -1 + if config.get('max_open_trades') == -1: + config['max_open_trades'] = float('inf') + + if self.args.get('stake_amount', None): + # Convert explicitly to float to support CLI argument for both unlimited and value + try: + self.args['stake_amount'] = float(self.args['stake_amount']) + except ValueError: + pass self._args_to_config(config, argname='stake_amount', logstring='Parameter --stake-amount detected, ' 'overriding stake_amount to: {} ...') - + self._args_to_config(config, argname='dry_run_wallet', + logstring='Parameter --dry-run-wallet detected, ' + 'overriding dry_run_wallet to: {} ...') self._args_to_config(config, argname='fee', logstring='Parameter --fee detected, ' 'setting fee to: {} ...') @@ -366,6 +375,9 @@ class Configuration: self._args_to_config(config, argname='plot_limit', logstring='Limiting plot to: {}') + self._args_to_config(config, argname='plot_auto_open', + logstring='Parameter --auto-open detected.') + self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') @@ -390,6 +402,11 @@ class Configuration: self._args_to_config(config, argname='dataformat_trades', logstring='Using "{}" to store trades data.') + def _process_data_options(self, config: Dict[str, Any]) -> None: + + self._args_to_config(config, argname='new_pairs_days', + logstring='Detected --new-pairs-days: {}') + def _process_runmode(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='dry_run', @@ -436,6 +453,7 @@ class Configuration: """ if "pairs" in config: + config['exchange']['pair_whitelist'] = config['pairs'] return if "pairs_file" in self.args and self.args["pairs_file"]: @@ -445,9 +463,8 @@ class Configuration: # or if pairs file is specified explicitely if not pairs_file.exists(): raise OperationalException(f'No pairs file found with path "{pairs_file}".') - with pairs_file.open('r') as f: - config['pairs'] = json_load(f) - config['pairs'].sort() + config['pairs'] = load_file(pairs_file) + config['pairs'].sort() return if 'config' in self.args and self.args['config']: @@ -457,7 +474,6 @@ class Configuration: # Fall back to /dl_path/pairs.json pairs_file = config['datadir'] / 'pairs.json' if pairs_file.exists(): - with pairs_file.open('r') as f: - config['pairs'] = json_load(f) + config['pairs'] = load_file(pairs_file) if 'pairs' in config: config['pairs'].sort() diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index 51310f013..ca305c260 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -24,6 +24,21 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Pat return folder +def chown_user_directory(directory: Path) -> None: + """ + Use Sudo to change permissions of the home-directory if necessary + Only applies when running in docker! + """ + import os + if os.environ.get('FT_APP_ENV') == 'docker': + try: + import subprocess + subprocess.check_output( + ['sudo', 'chown', '-R', 'ftuser:', str(directory.resolve())]) + except Exception: + logger.warning(f"Could not chown {directory}") + + def create_userdata_dir(directory: str, create_dir: bool = False) -> Path: """ Create userdata directory structure. @@ -37,6 +52,7 @@ def create_userdata_dir(directory: str, create_dir: bool = False) -> Path: sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "logs", "notebooks", "plot", "strategies", ] folder = Path(directory) + chown_user_directory(folder) if not folder.is_dir(): if create_dir: folder.mkdir(parents=True) @@ -72,6 +88,5 @@ def copy_sample_files(directory: Path, overwrite: bool = False) -> None: if not overwrite: logger.warning(f"File `{targetfile}` exists already, not deploying sample file.") continue - else: - logger.warning(f"File `{targetfile}` exists already, overwriting.") + logger.warning(f"File `{targetfile}` exists already, overwriting.") shutil.copy(str(sourcedir / source), str(targetfile)) diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 726126034..27190d259 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -38,6 +38,15 @@ def log_config_error_range(path: str, errmsg: str) -> str: return '' +def load_file(path: Path) -> Dict[str, Any]: + try: + with path.open('r') as file: + config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE) + except FileNotFoundError: + raise OperationalException(f'File "{path}" not found!') + return config + + def load_config_file(path: str) -> Dict[str, Any]: """ Loads a config file from the given path diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 32bbd02a0..6979c8cd1 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -3,10 +3,13 @@ This module contains the argument manager class """ import logging import re +from datetime import datetime from typing import Optional import arrow +from freqtrade.exceptions import OperationalException + logger = logging.getLogger(__name__) @@ -41,7 +44,7 @@ class TimeRange: self.startts = self.startts - seconds def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int, - min_date: arrow.Arrow) -> None: + min_date: datetime) -> None: """ Adjust startts by candles. Applies only if no startup-candles have been available. @@ -52,11 +55,11 @@ class TimeRange: :return: None (Modifies the object in place) """ if (not self.starttype or (startup_candles - and min_date.int_timestamp >= self.startts)): + and min_date.timestamp() >= self.startts)): # If no startts was defined, or backtest-data starts at the defined backtest-date logger.warning("Moving start-date by %s candles to account for startup time.", startup_candles) - self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles) + self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles) self.starttype = 'date' @staticmethod @@ -103,5 +106,8 @@ class TimeRange: stop = int(stops) // 1000 else: stop = int(stops) + if start > stop > 0: + raise OperationalException( + f'Start date is after stop date for timerange "{text}"') return TimeRange(stype[0], stype[1], start, stop) - raise Exception('Incorrect syntax for timerange "%s"' % text) + raise OperationalException(f'Incorrect syntax for timerange "{text}"') diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 802ddc2b1..e42b9d4b8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -11,6 +11,7 @@ DEFAULT_EXCHANGE = 'bittrex' PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec +TIMEOUT_UNITS = ['minutes', 'seconds'] DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' @@ -26,7 +27,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', - 'SpreadFilter'] + 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 @@ -54,6 +55,11 @@ DECIMALS_PER_COIN = { 'ETH': 5, } +DUST_PER_COIN = { + 'BTC': 0.0001, + 'ETH': 0.01 +} + # Soure files with destination directories within user-directory USER_DATA_FILES = { @@ -91,6 +97,7 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1}, + 'new_pairs_days': {'type': 'integer', 'default': 30}, 'timeframe': {'type': 'string'}, 'stake_currency': {'type': 'string'}, 'stake_amount': { @@ -131,7 +138,8 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'buy': {'type': 'number', 'minimum': 1}, - 'sell': {'type': 'number', 'minimum': 1} + 'sell': {'type': 'number', 'minimum': 1}, + 'unit': {'type': 'string', 'enum': TIMEOUT_UNITS, 'default': 'minutes'} } }, 'bid_strategy': { @@ -160,12 +168,18 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'ask'}, + 'bid_last_balance': { + 'type': 'number', + 'minimum': 0, + 'maximum': 1, + 'exclusiveMaximum': False, + }, 'use_order_book': {'type': 'boolean'}, 'order_book_min': {'type': 'integer', 'minimum': 1}, 'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50}, 'use_sell_signal': {'type': 'boolean'}, 'sell_profit_only': {'type': 'boolean'}, - 'sell_profit_offset': {'type': 'number', 'minimum': 0.0}, + 'sell_profit_offset': {'type': 'number'}, 'ignore_roi_if_buy_signal': {'type': 'boolean'} } }, @@ -174,6 +188,8 @@ CONF_SCHEMA = { 'properties': { 'buy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'sell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'forcesell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, + 'forcebuy': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'emergencysell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'stoploss_on_exchange': {'type': 'boolean'}, @@ -230,20 +246,37 @@ CONF_SCHEMA = { 'enabled': {'type': 'boolean'}, 'token': {'type': 'string'}, 'chat_id': {'type': 'string'}, + 'balance_dust_level': {'type': 'number', 'minimum': 0.0}, 'notification_settings': { 'type': 'object', + 'default': {}, 'properties': { 'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, - 'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS} + 'buy_fill': {'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + 'default': 'off' + }, + 'sell': { + 'type': ['string', 'object'], + 'additionalProperties': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS + } + }, + 'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'sell_fill': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS, + 'default': 'off' + }, } } }, - 'required': ['enabled', 'token', 'chat_id'] + 'required': ['enabled', 'token', 'chat_id'], }, 'webhook': { 'type': 'object', @@ -370,6 +403,16 @@ SCHEMA_TRADE_REQUIRED = [ 'dataformat_trades', ] +SCHEMA_BACKTEST_REQUIRED = [ + 'exchange', + 'max_open_trades', + 'stake_currency', + 'stake_amount', + 'dry_run_wallet', + 'dataformat_ohlcv', + 'dataformat_trades', +] + SCHEMA_MINIMAL_REQUIRED = [ 'exchange', 'dry_run', diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 828fb78f3..e7af5eab8 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -10,7 +10,7 @@ import pandas as pd from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.misc import json_load -from freqtrade.persistence import Trade, init_db +from freqtrade.persistence import LocalTrade, Trade, init_db logger = logging.getLogger(__name__) @@ -156,33 +156,35 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non data = data['strategy'][strategy]['trades'] df = pd.DataFrame(data) - df['open_date'] = pd.to_datetime(df['open_date'], - utc=True, - infer_datetime_format=True - ) - df['close_date'] = pd.to_datetime(df['close_date'], - utc=True, - infer_datetime_format=True - ) + if not df.empty: + df['open_date'] = pd.to_datetime(df['open_date'], + utc=True, + infer_datetime_format=True + ) + df['close_date'] = pd.to_datetime(df['close_date'], + utc=True, + infer_datetime_format=True + ) else: # old format - only with lists. df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD) - - df['open_date'] = pd.to_datetime(df['open_date'], - unit='s', - utc=True, - infer_datetime_format=True - ) - df['close_date'] = pd.to_datetime(df['close_date'], - unit='s', - utc=True, - infer_datetime_format=True - ) - # Create compatibility with new format - df['profit_abs'] = df['close_rate'] - df['open_rate'] - if 'profit_ratio' not in df.columns: - df['profit_ratio'] = df['profit_percent'] - df = df.sort_values("open_date").reset_index(drop=True) + if not df.empty: + df['open_date'] = pd.to_datetime(df['open_date'], + unit='s', + utc=True, + infer_datetime_format=True + ) + df['close_date'] = pd.to_datetime(df['close_date'], + unit='s', + utc=True, + infer_datetime_format=True + ) + # Create compatibility with new format + df['profit_abs'] = df['close_rate'] - df['open_rate'] + if not df.empty: + if 'profit_ratio' not in df.columns: + df['profit_ratio'] = df['profit_percent'] + df = df.sort_values("open_date").reset_index(drop=True) return df @@ -224,7 +226,7 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str, return df_final[df_final['open_trades'] > max_open_trades] -def trade_list_to_dataframe(trades: List[Trade]) -> pd.DataFrame: +def trade_list_to_dataframe(trades: List[LocalTrade]) -> pd.DataFrame: """ Convert list of Trade objects to pandas Dataframe :param trades: List of trade objects @@ -337,7 +339,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, """ Adds a column `col_name` with the cumulative profit for the given trades array. :param df: DataFrame with date index - :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) + :param trades: DataFrame containing trades (requires columns close_date and profit_abs) :param col_name: Column name that will be assigned the results :param timeframe: Timeframe used during the operations :return: Returns df with one additional column, col_name, containing the cumulative profit. @@ -349,8 +351,8 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, timeframe_minutes = timeframe_to_minutes(timeframe) # Resample to timeframe to make sure trades match candles _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' - )[['profit_ratio']].sum() - df.loc[:, col_name] = _trades_sum['profit_ratio'].cumsum() + )[['profit_abs']].sum() + df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum() # Set first value to 0 df.loc[df.iloc[0].name, col_name] = 0 # FFill to get continuous @@ -360,13 +362,14 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', value_col: str = 'profit_ratio' - ) -> Tuple[float, pd.Timestamp, pd.Timestamp]: + ) -> Tuple[float, pd.Timestamp, pd.Timestamp, float, float]: """ Calculate max drawdown and the corresponding close dates :param trades: DataFrame containing trades (requires columns close_date and profit_ratio) :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') :param value_col: Column in DataFrame to use for values (defaults to 'profit_ratio') - :return: Tuple (float, highdate, lowdate) with absolute max drawdown, high and low time + :return: Tuple (float, highdate, lowdate, highvalue, lowvalue) with absolute max drawdown, + high and low time and high and low value. :raise: ValueError if trade-dataframe was found empty. """ if len(trades) == 0: @@ -382,4 +385,26 @@ def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date' raise ValueError("No losing trade, therefore no drawdown.") high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] low_date = profit_results.loc[idxmin, date_col] - return abs(min(max_drawdown_df['drawdown'])), high_date, low_date + high_val = max_drawdown_df.loc[max_drawdown_df.iloc[:idxmin] + ['high_value'].idxmax(), 'cumulative'] + low_val = max_drawdown_df.loc[idxmin, 'cumulative'] + return abs(min(max_drawdown_df['drawdown'])), high_date, low_date, high_val, low_val + + +def calculate_csum(trades: pd.DataFrame, starting_balance: float = 0) -> Tuple[float, float]: + """ + Calculate min/max cumsum of trades, to show if the wallet/stake amount ratio is sane + :param trades: DataFrame containing trades (requires columns close_date and profit_percent) + :param starting_balance: Add starting balance to results, to show the wallets high / low points + :return: Tuple (float, float) with cumsum of profit_abs + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + + csum_df = pd.DataFrame() + csum_df['sum'] = trades['profit_abs'].cumsum() + csum_min = csum_df['sum'].min() + starting_balance + csum_max = csum_df['sum'].max() + starting_balance + + return csum_min, csum_max diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index d4053abaa..ffee0c52c 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -110,28 +110,62 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) df.reset_index(inplace=True) len_before = len(dataframe) len_after = len(df) + pct_missing = (len_after - len_before) / len_before if len_before > 0 else 0 if len_before != len_after: - logger.info(f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}") + message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}" + f" - {round(pct_missing * 100, 2)}%") + if pct_missing > 0.01: + logger.info(message) + else: + # Don't be verbose if only a small amount is missing + logger.debug(message) return df -def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date') -> DataFrame: +def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date', + startup_candles: int = 0) -> DataFrame: """ Trim dataframe based on given timerange :param df: Dataframe to trim :param timerange: timerange (use start and end date if available) - :param: df_date_col: Column in the dataframe to use as Date column + :param df_date_col: Column in the dataframe to use as Date column + :param startup_candles: When not 0, is used instead the timerange start date :return: trimmed dataframe """ - if timerange.starttype == 'date': - start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) - df = df.loc[df[df_date_col] >= start, :] + if startup_candles: + # Trim candles instead of timeframe in case of given startup_candle count + df = df.iloc[startup_candles:, :] + else: + if timerange.starttype == 'date': + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + df = df.loc[df[df_date_col] >= start, :] if timerange.stoptype == 'date': stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) df = df.loc[df[df_date_col] <= stop, :] return df +def trim_dataframes(preprocessed: Dict[str, DataFrame], timerange, + startup_candles: int) -> Dict[str, DataFrame]: + """ + Trim startup period from analyzed dataframes + :param preprocessed: Dict of pair: dataframe + :param timerange: timerange (use start and end date if available) + :param startup_candles: Startup-candles that should be removed + :return: Dict of trimmed dataframes + """ + processed: Dict[str, DataFrame] = {} + + for pair, df in preprocessed.items(): + trimed_df = trim_dataframe(df, timerange, startup_candles=startup_candles) + if not trimed_df.empty: + processed[pair] = trimed_df + else: + logger.warning(f'{pair} has no data left after adjusting for startup candles, ' + f'skipping.') + return processed + + def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: """ TODO: This should get a dedicated test diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index a035b7c3b..391ed4587 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -12,21 +12,32 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe from freqtrade.data.history import load_pair_history +from freqtrade.enums import RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.exchange import Exchange -from freqtrade.state import RunMode logger = logging.getLogger(__name__) +NO_EXCHANGE_EXCEPTION = 'Exchange is not available to DataProvider.' +MAX_DATAFRAME_CANDLES = 1000 + class DataProvider: - def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: + def __init__(self, config: dict, exchange: Optional[Exchange], pairlists=None) -> None: self._config = config self._exchange = exchange self._pairlists = pairlists self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} + self.__slice_index: Optional[int] = None + + def _set_dataframe_max_index(self, limit_index: int): + """ + Limit analyzed dataframe to max specified index. + :param limit_index: dataframe index. + """ + self.__slice_index = limit_index def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None: """ @@ -45,40 +56,6 @@ class DataProvider: """ self._pairlists = pairlists - def refresh(self, - pairlist: ListPairsWithTimeframes, - helping_pairs: ListPairsWithTimeframes = None) -> None: - """ - Refresh data, called with each cycle - """ - if helping_pairs: - self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) - else: - self._exchange.refresh_latest_ohlcv(pairlist) - - @property - def available_pairs(self) -> ListPairsWithTimeframes: - """ - Return a list of tuples containing (pair, timeframe) for which data is currently cached. - Should be whitelist + open trades. - """ - return list(self._exchange._klines.keys()) - - def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: - """ - Get candle (OHLCV) data for the given pair as DataFrame - Please use the `available_pairs` method to verify which pairs are currently cached. - :param pair: pair to get the data for - :param timeframe: Timeframe to get data for - :param copy: copy dataframe before returning if True. - Use False only for read-only operations (where the dataframe is not modified) - """ - if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - return self._exchange.klines((pair, timeframe or self._config['timeframe']), - copy=copy) - else: - return DataFrame() - def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame: """ Get stored historical candle (OHLCV) data @@ -111,47 +88,27 @@ class DataProvider: def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]: """ + Retrieve the analyzed dataframe. Returns the full dataframe in trade mode (live / dry), + and the last 1000 candles (up to the time evaluated at this moment) in all other modes. :param pair: pair to get the data for :param timeframe: timeframe to get data for :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe combination. Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. """ - if (pair, timeframe) in self.__cached_pairs: - return self.__cached_pairs[(pair, timeframe)] + pair_key = (pair, timeframe) + if pair_key in self.__cached_pairs: + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + df, date = self.__cached_pairs[pair_key] + else: + df, date = self.__cached_pairs[pair_key] + if self.__slice_index is not None: + max_index = self.__slice_index + df = df.iloc[max(0, max_index - MAX_DATAFRAME_CANDLES):max_index] + return df, date else: - return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) - def market(self, pair: str) -> Optional[Dict[str, Any]]: - """ - Return market data for the pair - :param pair: Pair to get the data for - :return: Market data dict from ccxt or None if market info is not available for the pair - """ - return self._exchange.markets.get(pair) - - def ticker(self, pair: str): - """ - Return last ticker data from exchange - :param pair: Pair to get the data for - :return: Ticker dict from exchange or empty dict if ticker is not available for the pair - """ - try: - return self._exchange.fetch_ticker(pair) - except ExchangeError: - return {} - - def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: - """ - Fetch latest l2 orderbook data - Warning: Does a network request - so use with common sense. - :param pair: pair to get the data for - :param maximum: Maximum number of orderbook entries to query - :return: dict including bids/asks with a total of `maximum` entries. - """ - return self._exchange.fetch_l2_order_book(pair, maximum) - @property def runmode(self) -> RunMode: """ @@ -170,6 +127,89 @@ class DataProvider: """ if self._pairlists: - return self._pairlists.whitelist + return self._pairlists.whitelist.copy() else: raise OperationalException("Dataprovider was not initialized with a pairlist provider.") + + def clear_cache(self): + """ + Clear pair dataframe cache. + """ + self.__cached_pairs = {} + + # Exchange functions + + def refresh(self, + pairlist: ListPairsWithTimeframes, + helping_pairs: ListPairsWithTimeframes = None) -> None: + """ + Refresh data, called with each cycle + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + if helping_pairs: + self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs) + else: + self._exchange.refresh_latest_ohlcv(pairlist) + + @property + def available_pairs(self) -> ListPairsWithTimeframes: + """ + Return a list of tuples containing (pair, timeframe) for which data is currently cached. + Should be whitelist + open trades. + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + return list(self._exchange._klines.keys()) + + def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: + """ + Get candle (OHLCV) data for the given pair as DataFrame + Please use the `available_pairs` method to verify which pairs are currently cached. + :param pair: pair to get the data for + :param timeframe: Timeframe to get data for + :param copy: copy dataframe before returning if True. + Use False only for read-only operations (where the dataframe is not modified) + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): + return self._exchange.klines((pair, timeframe or self._config['timeframe']), + copy=copy) + else: + return DataFrame() + + def market(self, pair: str) -> Optional[Dict[str, Any]]: + """ + Return market data for the pair + :param pair: Pair to get the data for + :return: Market data dict from ccxt or None if market info is not available for the pair + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + return self._exchange.markets.get(pair) + + def ticker(self, pair: str): + """ + Return last ticker data from exchange + :param pair: Pair to get the data for + :return: Ticker dict from exchange or empty dict if ticker is not available for the pair + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + try: + return self._exchange.fetch_ticker(pair) + except ExchangeError: + return {} + + def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: + """ + Fetch latest l2 orderbook data + Warning: Does a network request - so use with common sense. + :param pair: pair to get the data for + :param maximum: Maximum number of orderbook entries to query + :return: dict including bids/asks with a total of `maximum` entries. + """ + if self._exchange is None: + raise OperationalException(NO_EXCHANGE_EXCEPTION) + return self._exchange.fetch_l2_order_book(pair, maximum) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index d116637e7..e80cfeba2 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -89,7 +89,7 @@ class HDF5DataHandler(IDataHandler): if timerange.starttype == 'date': where.append(f"date >= Timestamp({timerange.startts * 1e9})") if timerange.stoptype == 'date': - where.append(f"date < Timestamp({timerange.stopts * 1e9})") + where.append(f"date <= Timestamp({timerange.stopts * 1e9})") pairdata = pd.read_hdf(filename, key=key, mode="r", where=where) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 3b8b5a2f0..86e9f75e6 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -155,6 +155,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona def _download_pair_history(datadir: Path, exchange: Exchange, pair: str, *, + new_pairs_days: int = 30, timeframe: str = '5m', timerange: Optional[TimeRange] = None, data_handler: IDataHandler = None) -> bool: @@ -193,7 +194,7 @@ def _download_pair_history(datadir: Path, timeframe=timeframe, since_ms=since_ms if since_ms else int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000 + days=-new_pairs_days).float_timestamp) * 1000 ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, @@ -223,7 +224,8 @@ def _download_pair_history(datadir: Path, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], datadir: Path, timerange: Optional[TimeRange] = None, - erase: bool = False, data_format: str = None) -> List[str]: + new_pairs_days: int = 30, erase: bool = False, + data_format: str = None) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -246,12 +248,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes logger.info(f'Downloading pair {pair}, interval {timeframe}.') _download_pair_history(datadir=datadir, exchange=exchange, pair=pair, timeframe=str(timeframe), + new_pairs_days=new_pairs_days, timerange=timerange, data_handler=data_handler) return pairs_not_available def _download_trades_history(exchange: Exchange, pair: str, *, + new_pairs_days: int = 30, timerange: Optional[TimeRange] = None, data_handler: IDataHandler ) -> bool: @@ -261,9 +265,13 @@ def _download_trades_history(exchange: Exchange, """ try: - since = timerange.startts * 1000 if \ - (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000 + until = None + if (timerange and timerange.starttype == 'date'): + since = timerange.startts * 1000 + if timerange.stoptype == 'date': + until = timerange.stopts * 1000 + else: + since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000 trades = data_handler.trades_load(pair) @@ -291,6 +299,7 @@ def _download_trades_history(exchange: Exchange, # Default since_ms to 30 days if nothing is given new_trades = exchange.get_historic_trades(pair=pair, since=since, + until=until, from_id=from_id, ) trades.extend(new_trades[1]) @@ -311,8 +320,8 @@ def _download_trades_history(exchange: Exchange, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, erase: bool = False, - data_format: str = 'jsongz') -> List[str]: + timerange: TimeRange, new_pairs_days: int = 30, + erase: bool = False, data_format: str = 'jsongz') -> List[str]: """ Refresh stored trades data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -333,6 +342,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: logger.info(f'Downloading trades for pair {pair}.') _download_trades_history(exchange=exchange, pair=pair, + new_pairs_days=new_pairs_days, timerange=timerange, data_handler=data_handler) return pairs_not_available @@ -362,7 +372,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], logger.exception(f'Could not convert {pair} to OHLCV.') -def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: +def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]: """ Get the maximum common timerange for the given backtest data. @@ -370,7 +380,7 @@ def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow] :return: tuple containing min_date, max_date """ timeranges = [ - (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + (frame['date'].min().to_pydatetime(), frame['date'].max().to_pydatetime()) for frame in data.values() ] return (min(timeranges, key=operator.itemgetter(0))[0], diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 2bdef1c89..9466a1649 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -1,6 +1,8 @@ # pragma pylint: disable=W0603 """ Edge positioning package """ import logging +from collections import defaultdict +from copy import deepcopy from typing import Any, Dict, List, NamedTuple import arrow @@ -11,9 +13,11 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT from freqtrade.data.history import get_timerange, load_data, refresh_data +from freqtrade.enums import RunMode, SellType from freqtrade.exceptions import OperationalException +from freqtrade.exchange.exchange import timeframe_to_seconds from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist -from freqtrade.strategy.interface import SellType +from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) @@ -45,7 +49,7 @@ class Edge: self.config = config self.exchange = exchange - self.strategy = strategy + self.strategy: IStrategy = strategy self.edge_config = self.config.get('edge', {}) self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs @@ -81,12 +85,16 @@ class Edge: if config.get('fee'): self.fee = config['fee'] else: - self.fee = self.exchange.get_fee(symbol=expand_pairlist( - self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0]) + try: + self.fee = self.exchange.get_fee(symbol=expand_pairlist( + self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0]) + except IndexError: + self.fee = None + + def calculate(self, pairs: List[str]) -> bool: + if self.fee is None and pairs: + self.fee = self.exchange.get_fee(pairs[0]) - def calculate(self) -> bool: - pairs = expand_pairlist(self.config['exchange']['pair_whitelist'], - list(self.exchange.markets)) heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated > 0) and ( @@ -98,13 +106,33 @@ class Edge: logger.info('Using local backtesting data (using whitelist in given config) ...') if self._refresh_pairs: + timerange_startup = deepcopy(self._timerange) + timerange_startup.subtract_start(timeframe_to_seconds( + self.strategy.timeframe) * self.strategy.startup_candle_count) refresh_data( datadir=self.config['datadir'], pairs=pairs, exchange=self.exchange, timeframe=self.strategy.timeframe, - timerange=self._timerange, + timerange=timerange_startup, + data_format=self.config.get('dataformat_ohlcv', 'json'), ) + # Download informative pairs too + res = defaultdict(list) + for p, t in self.strategy.informative_pairs(): + res[t].append(p) + for timeframe, inf_pairs in res.items(): + timerange_startup = deepcopy(self._timerange) + timerange_startup.subtract_start(timeframe_to_seconds( + timeframe) * self.strategy.startup_candle_count) + refresh_data( + datadir=self.config['datadir'], + pairs=inf_pairs, + exchange=self.exchange, + timeframe=timeframe, + timerange=timerange_startup, + data_format=self.config.get('dataformat_ohlcv', 'json'), + ) data = load_data( datadir=self.config['datadir'], @@ -120,8 +148,11 @@ class Edge: self._cached_pairs = {} logger.critical("No data found. Edge is stopped ...") return False - + # Fake run-mode to Edge + prior_rm = self.config['runmode'] + self.config['runmode'] = RunMode.EDGE preprocessed = self.strategy.ohlcvdata_to_dataframe(data) + self.config['runmode'] = prior_rm # Print timeframe min_date, max_date = get_timerange(preprocessed) @@ -178,7 +209,7 @@ class Edge: if pair in self._cached_pairs: return self._cached_pairs[pair].stoploss else: - logger.warning('tried to access stoploss of a non-existing pair, ' + logger.warning(f'Tried to access stoploss of non-existing pair {pair}, ' 'strategy stoploss is returned instead.') return self.strategy.stoploss @@ -209,7 +240,7 @@ class Edge: return self._final_pairs - def accepted_pairs(self) -> list: + def accepted_pairs(self) -> List[Dict[str, Any]]: """ return a list of accepted pairs along with their winrate, expectancy and stoploss """ diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py new file mode 100644 index 000000000..78163d86f --- /dev/null +++ b/freqtrade/enums/__init__.py @@ -0,0 +1,6 @@ +# flake8: noqa: F401 +from freqtrade.enums.rpcmessagetype import RPCMessageType +from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode +from freqtrade.enums.selltype import SellType +from freqtrade.enums.signaltype import SignalType +from freqtrade.enums.state import State diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py new file mode 100644 index 000000000..9c59f6108 --- /dev/null +++ b/freqtrade/enums/rpcmessagetype.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class RPCMessageType(Enum): + STATUS = 'status' + WARNING = 'warning' + STARTUP = 'startup' + BUY = 'buy' + BUY_FILL = 'buy_fill' + BUY_CANCEL = 'buy_cancel' + SELL = 'sell' + SELL_FILL = 'sell_fill' + SELL_CANCEL = 'sell_cancel' + + def __repr__(self): + return self.value + + def __str__(self): + return self.value diff --git a/freqtrade/state.py b/freqtrade/enums/runmode.py similarity index 68% rename from freqtrade/state.py rename to freqtrade/enums/runmode.py index 8ddff71d9..7826d1d0c 100644 --- a/freqtrade/state.py +++ b/freqtrade/enums/runmode.py @@ -1,23 +1,6 @@ -# pragma pylint: disable=too-few-public-methods - -""" -Bot state constant -""" from enum import Enum -class State(Enum): - """ - Bot application states - """ - RUNNING = 1 - STOPPED = 2 - RELOAD_CONFIG = 3 - - def __str__(self): - return f"{self.name.lower()}" - - class RunMode(Enum): """ Bot running mode (backtest, hyperopt, ...) diff --git a/freqtrade/enums/selltype.py b/freqtrade/enums/selltype.py new file mode 100644 index 000000000..015c30186 --- /dev/null +++ b/freqtrade/enums/selltype.py @@ -0,0 +1,20 @@ +from enum import Enum + + +class SellType(Enum): + """ + Enum to distinguish between sell reasons + """ + ROI = "roi" + STOP_LOSS = "stop_loss" + STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange" + TRAILING_STOP_LOSS = "trailing_stop_loss" + SELL_SIGNAL = "sell_signal" + FORCE_SELL = "force_sell" + EMERGENCY_SELL = "emergency_sell" + CUSTOM_SELL = "custom_sell" + NONE = "" + + def __str__(self): + # explicitly convert to String to help with exporting data. + return self.value diff --git a/freqtrade/enums/signaltype.py b/freqtrade/enums/signaltype.py new file mode 100644 index 000000000..d636f378a --- /dev/null +++ b/freqtrade/enums/signaltype.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class SignalType(Enum): + """ + Enum to distinguish between buy and sell signals + """ + BUY = "buy" + SELL = "sell" diff --git a/freqtrade/enums/state.py b/freqtrade/enums/state.py new file mode 100644 index 000000000..572e2299f --- /dev/null +++ b/freqtrade/enums/state.py @@ -0,0 +1,13 @@ +from enum import Enum + + +class State(Enum): + """ + Bot application states + """ + RUNNING = 1 + STOPPED = 2 + RELOAD_CONFIG = 3 + + def __str__(self): + return f"{self.name.lower()}" diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 15ba7b9f6..015e0c869 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -7,11 +7,14 @@ from freqtrade.exchange.bibox import Bibox from freqtrade.exchange.binance import Binance from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.bybit import Bybit +from freqtrade.exchange.coinbasepro import Coinbasepro from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges, - get_exchange_bad_reason, is_exchange_bad, is_exchange_known_ccxt, is_exchange_officially_supported, market_is_active, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, - timeframe_to_seconds) + timeframe_to_seconds, validate_exchange, + validate_exchanges) from freqtrade.exchange.ftx import Ftx +from freqtrade.exchange.hitbtc import Hitbtc from freqtrade.exchange.kraken import Kraken +from freqtrade.exchange.kucoin import Kucoin diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 26ec30a8a..0bcfa5e17 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -52,7 +52,7 @@ class Binance(Exchange): 'In stoploss limit order, stop price should be more than limit price') if self._config['dry_run']: - dry_order = self.dry_run_order( + dry_order = self.create_dry_run_order( pair, ordertype, "sell", amount, stop_price) return dry_order diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py index 4318f9cf0..69e2f2b8d 100644 --- a/freqtrade/exchange/bittrex.py +++ b/freqtrade/exchange/bittrex.py @@ -12,12 +12,14 @@ class Bittrex(Exchange): """ Bittrex exchange class. Contains adjustments needed for Freqtrade to work with this exchange. - - Please note that this exchange is not included in the list of exchanges - officially supported by the Freqtrade development team. So some features - may still not work as expected. """ _ft_has: Dict = { + "ohlcv_candle_limit_per_timeframe": { + '1m': 1440, + '5m': 288, + '1h': 744, + '1d': 365, + }, "l2_limit_range": [1, 25, 500], } diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py index 4a44bb42d..163f8c44e 100644 --- a/freqtrade/exchange/bybit.py +++ b/freqtrade/exchange/bybit.py @@ -18,7 +18,6 @@ class Bybit(Exchange): may still not work as expected. """ - # fetchCurrencies API point requires authentication for Bybit, _ft_has: Dict = { "ohlcv_candle_limit": 200, } diff --git a/freqtrade/exchange/coinbasepro.py b/freqtrade/exchange/coinbasepro.py new file mode 100644 index 000000000..7dd9c80dc --- /dev/null +++ b/freqtrade/exchange/coinbasepro.py @@ -0,0 +1,23 @@ +""" CoinbasePro exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Coinbasepro(Exchange): + """ + CoinbasePro exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + _ft_has: Dict = { + "ohlcv_candle_limit": 300, + } diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index c66db860f..694aa3aa2 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -18,78 +18,8 @@ BAD_EXCHANGES = { "bitmex": "Various reasons.", "bitstamp": "Does not provide history. " "Details in https://github.com/freqtrade/freqtrade/issues/1983", - "hitbtc": "This API cannot be used with Freqtrade. " - "Use `hitbtc2` exchange id to access this exchange.", "phemex": "Does not provide history. ", "poloniex": "Does not provide fetch_order endpoint to fetch both open and closed orders.", - **dict.fromkeys([ - 'adara', - 'anxpro', - 'bigone', - 'coinbase', - 'coinexchange', - 'coinmarketcap', - 'lykke', - 'xbtce', - ], "Does not provide timeframes. ccxt fetchOHLCV: False"), - **dict.fromkeys([ - 'bcex', - 'bit2c', - 'bitbay', - 'bitflyer', - 'bitforex', - 'bithumb', - 'bitso', - 'bitstamp1', - 'bl3p', - 'braziliex', - 'btcbox', - 'btcchina', - 'btctradeim', - 'btctradeua', - 'bxinth', - 'chilebit', - 'coincheck', - 'coinegg', - 'coinfalcon', - 'coinfloor', - 'coingi', - 'coinmate', - 'coinone', - 'coinspot', - 'coolcoin', - 'crypton', - 'deribit', - 'exmo', - 'exx', - 'flowbtc', - 'foxbit', - 'fybse', - # 'hitbtc', - 'ice3x', - 'independentreserve', - 'indodax', - 'itbit', - 'lakebtc', - 'latoken', - 'liquid', - 'livecoin', - 'luno', - 'mixcoins', - 'negociecoins', - 'nova', - 'paymium', - 'southxchange', - 'stronghold', - 'surbitcoin', - 'therock', - 'tidex', - 'vaultoro', - 'vbtc', - 'virwox', - 'yobit', - 'zaif', - ], "Does not provide timeframes. ccxt fetchOHLCV: emulated"), } MAP_EXCHANGE_CHILDCLASS = { @@ -98,6 +28,29 @@ MAP_EXCHANGE_CHILDCLASS = { } +EXCHANGE_HAS_REQUIRED = [ + # Required / private + 'fetchOrder', + 'cancelOrder', + 'createOrder', + # 'createLimitOrder', 'createMarketOrder', + 'fetchBalance', + + # Public endpoints + 'loadMarkets', + 'fetchOHLCV', +] + +EXCHANGE_HAS_OPTIONAL = [ + # Private + 'fetchMyTrades', # Trades for order - fee detection + # Public + 'fetchOrderBook', 'fetchL2OrderBook', 'fetchTicker', # OR for pricing + 'fetchTickers', # For volumepairlist? + 'fetchTrades', # Downloading trades data +] + + def calculate_backoff(retrycount, max_retries): """ Calculate backoff @@ -140,7 +93,7 @@ def retrier(_func=None, retries=API_RETRY_COUNT): logger.warning('retrying %s() still for %s times', f.__name__, count) count -= 1 kwargs.update({'count': count}) - if isinstance(ex, DDosProtection) or isinstance(ex, RetryableOrderError): + if isinstance(ex, (DDosProtection, RetryableOrderError)): # increasing backoff backoff_delay = calculate_backoff(count + 1, retries) logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}") diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c7625b53c..67676d4e0 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -3,6 +3,7 @@ Cryptocurrency Exchanges support """ import asyncio +import http import inspect import logging from copy import deepcopy @@ -13,6 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple import arrow import ccxt import ccxt.async_support as ccxt_async +from cachetools import TTLCache from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision) from pandas import DataFrame @@ -20,9 +22,10 @@ from pandas import DataFrame from freqtrade.constants import DEFAULT_AMOUNT_RESERVE_PERCENT, ListPairsWithTimeframes from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, - InvalidOrderException, OperationalException, RetryableOrderError, - TemporaryError) -from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier, + InvalidOrderException, OperationalException, PricingError, + RetryableOrderError, TemporaryError) +from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, + EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, retrier_async) from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -34,6 +37,12 @@ CcxtModuleType = Any logger = logging.getLogger(__name__) +# Workaround for adding samesite support to pre 3.8 python +# Only applies to python3.7, and only on certain exchanges (kraken) +# Replicates the fix from starlette (which is actually causing this problem) +http.cookies.Morsel._reserved["samesite"] = "SameSite" # type: ignore + + class Exchange: _config: Dict = {} @@ -50,11 +59,13 @@ class Exchange: _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, "trades_pagination": "time", # Possible are "time" or "id" "trades_pagination_arg": "since", "l2_limit_range": None, + "l2_limit_range_required": True, # Allow Empty L2 limit (kucoin) } _ft_has: Dict = {} @@ -75,6 +86,14 @@ class Exchange: # Timestamp of last markets refresh self._last_markets_refresh: int = 0 + # Cache for 10 minutes ... + self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10) + # Cache values for 1800 to avoid frequent polling of the exchange for prices + # Caching only applies to RPC methods, so prices for open trades are still + # refreshed once every iteration. + self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) + self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) + # Holds candles self._klines: Dict[Tuple[str, str], DataFrame] = {} @@ -94,7 +113,6 @@ class Exchange: logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has) # Assign this directly for easy access - self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit'] self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle'] self._trades_pagination = self._ft_has['trades_pagination'] @@ -130,7 +148,8 @@ class Exchange: self.validate_pairs(config['exchange']['pair_whitelist']) self.validate_ordertypes(config.get('order_types', {})) self.validate_order_time_in_force(config.get('order_time_in_force', {})) - self.validate_required_startup_candles(config.get('startup_candle_count', 0)) + self.validate_required_startup_candles(config.get('startup_candle_count', 0), + config.get('timeframe', '')) # Converts the interval provided in minutes in config to seconds self.markets_refresh_interval: int = exchange_config.get( @@ -140,6 +159,9 @@ class Exchange: """ Destructor - clean up async stuff """ + self.close() + + def close(self): logger.debug("Exchange object destroyed, closing async loop") if self._api_async and inspect.iscoroutinefunction(self._api_async.close): asyncio.get_event_loop().run_until_complete(self._api_async.close()) @@ -191,11 +213,6 @@ class Exchange: def timeframes(self) -> List[str]: return list((self._api.timeframes or {}).keys()) - @property - def ohlcv_candle_limit(self) -> int: - """exchange ohlcv candle limit""" - return int(self._ohlcv_candle_limit) - @property def markets(self) -> Dict: """exchange ccxt markets""" @@ -209,6 +226,17 @@ class Exchange: """exchange ccxt precisionMode""" return self._api.precisionMode + def ohlcv_candle_limit(self, timeframe: str) -> int: + """ + Exchange ohlcv candle limit + Uses ohlcv_candle_limit_per_timeframe if the exchange has different limts + per timeframe (e.g. bittrex), otherwise falls back to ohlcv_candle_limit + :param timeframe: Timeframe to check + :return: Candle limit as integer + """ + return int(self._ft_has.get('ohlcv_candle_limit_per_timeframe', {}).get( + timeframe, self._ft_has.get('ohlcv_candle_limit'))) + def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None, pairs_only: bool = False, active_only: bool = False) -> Dict[str, Any]: """ @@ -295,8 +323,8 @@ class Exchange: self._markets = self._api.load_markets() self._load_async_markets() self._last_markets_refresh = arrow.utcnow().int_timestamp - except ccxt.BaseError as e: - logger.warning('Unable to initialize markets. Reason: %s', e) + except ccxt.BaseError: + logger.exception('Unable to initialize markets.') def reload_markets(self) -> None: """Reload markets both sync and async if refresh interval has passed """ @@ -341,7 +369,6 @@ class Exchange: invalid_pairs = [] for pair in extended_pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs - # TODO: add a support for having coins in BTC/USDT format if self.markets and pair not in self.markets: raise OperationalException( f'Pair {pair} is not available on {self.name}. ' @@ -421,15 +448,16 @@ class Exchange: raise OperationalException( f'Time in force policies are not supported for {self.name} yet.') - def validate_required_startup_candles(self, startup_candles: int) -> None: + def validate_required_startup_candles(self, startup_candles: int, timeframe: str) -> None: """ - Checks if required startup_candles is more than ohlcv_candle_limit. + Checks if required startup_candles is more than ohlcv_candle_limit(). Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. """ - if startup_candles + 5 > self._ft_has['ohlcv_candle_limit']: + candle_limit = self.ohlcv_candle_limit(timeframe) + if startup_candles + 5 > candle_limit: raise OperationalException( f"This strategy requires {startup_candles} candles to start. " - f"{self.name} only provides {self._ft_has['ohlcv_candle_limit']}.") + f"{self.name} only provides {candle_limit} for {timeframe}.") def exchange_has(self, endpoint: str) -> bool: """ @@ -443,7 +471,7 @@ class Exchange: def amount_to_precision(self, pair: str, amount: float) -> float: ''' Returns the amount to buy or sell to a precision the Exchange accepts - Reimplementation of ccxt internal methods - ensuring we can test the result is correct + Re-implementation of ccxt internal methods - ensuring we can test the result is correct based on our definitions. ''' if self.markets[pair]['precision']['amount']: @@ -457,7 +485,7 @@ class Exchange: def price_to_precision(self, pair: str, price: float) -> float: ''' Returns the price rounded up to the precision the Exchange accepts. - Partial Reimplementation of ccxt internal method decimal_to_precision(), + Partial Re-implementation of ccxt internal method decimal_to_precision(), which does not support rounding up TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and align with amount_to_precision(). @@ -514,19 +542,23 @@ class Exchange: return None # reserve some percent defined in config (5% default) + stoploss - amount_reserve_percent = 1.0 - self._config.get('amount_reserve_percent', + amount_reserve_percent = 1.0 + self._config.get('amount_reserve_percent', DEFAULT_AMOUNT_RESERVE_PERCENT) - amount_reserve_percent += stoploss + amount_reserve_percent = ( + amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5 + ) # it should not be more than 50% - amount_reserve_percent = max(amount_reserve_percent, 0.5) + amount_reserve_percent = max(min(amount_reserve_percent, 1.5), 1) # The value returned should satisfy both limits: for amount (base currency) and # for cost (quote, stake currency), so max() is used here. # See also #2575 at github. - return max(min_stake_amounts) / amount_reserve_percent + return max(min_stake_amounts) * amount_reserve_percent - def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, - rate: float, params: Dict = {}) -> Dict[str, Any]: + # Dry-run methods + + def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, + rate: float, params: Dict = {}) -> Dict[str, Any]: order_id = f'dry_run_{side}_{datetime.now().timestamp()}' _amount = self.amount_to_precision(pair, amount) dry_order = { @@ -566,6 +598,21 @@ class Exchange: closed_order["info"].update({"stopPrice": closed_order["price"]}) self._dry_run_open_orders[closed_order["id"]] = closed_order + def fetch_dry_run_order(self, order_id) -> Dict[str, Any]: + """ + Return dry-run order + Only call if running in dry-run mode. + """ + try: + order = self._dry_run_open_orders[order_id] + return order + except KeyError as e: + # Gracefully handle errors with dry-run orders. + raise InvalidOrderException( + f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + + # Order handling + def create_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict: try: @@ -600,7 +647,7 @@ class Exchange: rate: float, time_in_force: str) -> Dict: if self._config['dry_run']: - dry_order = self.dry_run_order(pair, ordertype, "buy", amount, rate) + dry_order = self.create_dry_run_order(pair, ordertype, "buy", amount, rate) return dry_order params = self._params.copy() @@ -613,7 +660,7 @@ class Exchange: rate: float, time_in_force: str = 'gtc') -> Dict: if self._config['dry_run']: - dry_order = self.dry_run_order(pair, ordertype, "sell", amount, rate) + dry_order = self.create_dry_run_order(pair, ordertype, "sell", amount, rate) return dry_order params = self._params.copy() @@ -642,396 +689,39 @@ class Exchange: raise OperationalException(f"stoploss is not implemented for {self.name}.") - @retrier - def get_balance(self, currency: str) -> float: + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) + def fetch_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - return self._config['dry_run_wallet'] - - # ccxt exception is already handled by get_balances - balances = self.get_balances() - balance = balances.get(currency) - if balance is None: - raise TemporaryError( - f'Could not get {currency} balance due to malformed exchange response: {balances}') - return balance['free'] - - @retrier - def get_balances(self) -> dict: - if self._config['dry_run']: - return {} - + return self.fetch_dry_run_order(order_id) try: - balances = self._api.fetch_balance() - # Remove additional info from ccxt results - balances.pop("info", None) - balances.pop("free", None) - balances.pop("total", None) - balances.pop("used", None) - - return balances + return self._api.fetch_order(order_id, pair) + except ccxt.OrderNotFound as e: + raise RetryableOrderError( + f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( - f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e + f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e - @retrier - def get_tickers(self) -> Dict: - try: - return self._api.fetch_tickers() - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching tickers in batch. ' - f'Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load tickers due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e + # Assign method to fetch_stoploss_order to allow easy overriding in other classes + fetch_stoploss_order = fetch_order - @retrier - def fetch_ticker(self, pair: str) -> dict: - try: - if (pair not in self.markets or - self.markets[pair].get('active', False) is False): - raise ExchangeError(f"Pair {pair} not available") - data = self._api.fetch_ticker(pair) - return data - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int) -> List: + def fetch_order_or_stoploss_order(self, order_id: str, pair: str, + stoploss_order: bool = False) -> Dict: """ - Get candle history using asyncio and returns the list of candles. - Handles all async work for this. - Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call. - :param pair: Pair to download - :param timeframe: Timeframe to get data for - :param since_ms: Timestamp in milliseconds to get history from - :return: List with candle (OHLCV) data + Simple wrapper calling either fetch_order or fetch_stoploss_order depending on + the stoploss_order parameter + :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. """ - return asyncio.get_event_loop().run_until_complete( - self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms)) - - def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, - since_ms: int) -> DataFrame: - """ - Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe - :param pair: Pair to download - :param timeframe: Timeframe to get data for - :param since_ms: Timestamp in milliseconds to get history from - :return: OHLCV DataFrame - """ - ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) - return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) - - async def _async_get_historic_ohlcv(self, pair: str, - timeframe: str, - since_ms: int) -> List: - """ - Download historic ohlcv - """ - - one_call = timeframe_to_msecs(timeframe) * self._ohlcv_candle_limit - logger.debug( - "one_call: %s msecs (%s)", - one_call, - arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) - ) - input_coroutines = [self._async_get_candle_history( - pair, timeframe, since) for since in - range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - - results = await asyncio.gather(*input_coroutines, return_exceptions=True) - - # Combine gathered results - data: List = [] - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: - data.extend(new_data) - # Sort data again after extending the result - above calls return in "async order" - data = sorted(data, key=lambda x: x[0]) - logger.info("Downloaded data for %s with length %s.", pair, len(data)) - return data - - def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, - since_ms: Optional[int] = None, cache: bool = True - ) -> Dict[Tuple[str, str], DataFrame]: - """ - Refresh in-memory OHLCV asynchronously and set `_klines` with the result - Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). - Only used in the dataprovider.refresh() method. - :param pair_list: List of 2 element tuples containing pair, interval to refresh - :param since_ms: time since when to download, in milliseconds - :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists - :return: Dict of [{(pair, timeframe): Dataframe}] - """ - logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) - - input_coroutines = [] - - # Gather coroutines to run - for pair, timeframe in set(pair_list): - if (not ((pair, timeframe) in self._klines) - or self._now_is_time_to_refresh(pair, timeframe)): - input_coroutines.append(self._async_get_candle_history(pair, timeframe, - since_ms=since_ms)) - else: - logger.debug( - "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", - pair, timeframe - ) - - results = asyncio.get_event_loop().run_until_complete( - asyncio.gather(*input_coroutines, return_exceptions=True)) - - results_df = {} - # handle caching - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple (has 3 elements) - pair, timeframe, ticks = res - # keeping last candle time as last refreshed time of the pair - if ticks: - self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 - # keeping parsed dataframe in cache - ohlcv_df = ohlcv_to_dataframe( - ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) - results_df[(pair, timeframe)] = ohlcv_df - if cache: - self._klines[(pair, timeframe)] = ohlcv_df - return results_df - - def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool: - # Timeframe in seconds - interval_in_sec = timeframe_to_seconds(timeframe) - - return not ((self._pairs_last_refresh_time.get((pair, timeframe), 0) - + interval_in_sec) >= arrow.utcnow().int_timestamp) - - @retrier_async - async def _async_get_candle_history(self, pair: str, timeframe: str, - since_ms: Optional[int] = None) -> Tuple[str, str, List]: - """ - Asynchronously get candle history data using fetch_ohlcv - returns tuple: (pair, timeframe, ohlcv_list) - """ - try: - # Fetch OHLCV asynchronously - s = '(' + arrow.get(since_ms // 1000).isoformat() + ') ' if since_ms is not None else '' - logger.debug( - "Fetching pair %s, interval %s, since %s %s...", - pair, timeframe, since_ms, s - ) - - data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, - since=since_ms, - limit=self._ohlcv_candle_limit) - - # Some exchanges sort OHLCV in ASC order and others in DESC. - # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) - # while GDAX returns the list of OHLCV in DESC order (newest first, oldest last) - # Only sort if necessary to save computing time - try: - if data and data[0][0] > data[-1][0]: - data = sorted(data, key=lambda x: x[0]) - except IndexError: - logger.exception("Error loading %s. Result was %s.", pair, data) - return pair, timeframe, [] - logger.debug("Done fetching pair %s, interval %s ...", pair, timeframe) - return pair, timeframe, data - - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching historical ' - f'candle (OHLCV) data. Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError(f'Could not fetch historical candle (OHLCV) data ' - f'for pair {pair} due to {e.__class__.__name__}. ' - f'Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch historical candle (OHLCV) data ' - f'for pair {pair}. Message: {e}') from e - - @retrier_async - async def _async_fetch_trades(self, pair: str, - since: Optional[int] = None, - params: Optional[dict] = None) -> List[List]: - """ - Asyncronously gets trade history using fetch_trades. - Handles exchange errors, does one call to the exchange. - :param pair: Pair to fetch trade data for - :param since: Since as integer timestamp in milliseconds - returns: List of dicts containing trades - """ - try: - # fetch trades asynchronously - if params: - logger.debug("Fetching trades for pair %s, params: %s ", pair, params) - trades = await self._api_async.fetch_trades(pair, params=params, limit=1000) - else: - logger.debug( - "Fetching trades for pair %s, since %s %s...", - pair, since, - '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else '' - ) - trades = await self._api_async.fetch_trades(pair, since=since, limit=1000) - return trades_dict_to_list(trades) - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching historical trade data.' - f'Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError(f'Could not load trade history due to {e.__class__.__name__}. ' - f'Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e - - async def _async_get_trade_history_id(self, pair: str, - until: int, - since: Optional[int] = None, - from_id: Optional[str] = None) -> Tuple[str, List[List]]: - """ - Asyncronously gets trade history using fetch_trades - use this when exchange uses id-based iteration (check `self._trades_pagination`) - :param pair: Pair to fetch trade data for - :param since: Since as integer timestamp in milliseconds - :param until: Until as integer timestamp in milliseconds - :param from_id: Download data starting with ID (if id is known). Ignores "since" if set. - returns tuple: (pair, trades-list) - """ - - trades: List[List] = [] - - if not from_id: - # Fetch first elements using timebased method to get an ID to paginate on - # Depending on the Exchange, this can introduce a drift at the start of the interval - # of up to an hour. - # e.g. Binance returns the "last 1000" candles within a 1h time interval - # - so we will miss the first trades. - t = await self._async_fetch_trades(pair, since=since) - # DEFAULT_TRADES_COLUMNS: 0 -> timestamp - # DEFAULT_TRADES_COLUMNS: 1 -> id - from_id = t[-1][1] - trades.extend(t[:-1]) - while True: - t = await self._async_fetch_trades(pair, - params={self._trades_pagination_arg: from_id}) - if len(t): - # Skip last id since its the key for the next call - trades.extend(t[:-1]) - if from_id == t[-1][1] or t[-1][0] > until: - logger.debug(f"Stopping because from_id did not change. " - f"Reached {t[-1][0]} > {until}") - # Reached the end of the defined-download period - add last trade as well. - trades.extend(t[-1:]) - break - - from_id = t[-1][1] - else: - break - - return (pair, trades) - - async def _async_get_trade_history_time(self, pair: str, until: int, - since: Optional[int] = None) -> Tuple[str, List[List]]: - """ - Asyncronously gets trade history using fetch_trades, - when the exchange uses time-based iteration (check `self._trades_pagination`) - :param pair: Pair to fetch trade data for - :param since: Since as integer timestamp in milliseconds - :param until: Until as integer timestamp in milliseconds - returns tuple: (pair, trades-list) - """ - - trades: List[List] = [] - # DEFAULT_TRADES_COLUMNS: 0 -> timestamp - # DEFAULT_TRADES_COLUMNS: 1 -> id - while True: - t = await self._async_fetch_trades(pair, since=since) - if len(t): - since = t[-1][0] - trades.extend(t) - # Reached the end of the defined-download period - if until and t[-1][0] > until: - logger.debug( - f"Stopping because until was reached. {t[-1][0]} > {until}") - break - else: - break - - return (pair, trades) - - async def _async_get_trade_history(self, pair: str, - since: Optional[int] = None, - until: Optional[int] = None, - from_id: Optional[str] = None) -> Tuple[str, List[List]]: - """ - Async wrapper handling downloading trades using either time or id based methods. - """ - - logger.debug(f"_async_get_trade_history(), pair: {pair}, " - f"since: {since}, until: {until}, from_id: {from_id}") - - if until is None: - until = ccxt.Exchange.milliseconds() - logger.debug(f"Exchange milliseconds: {until}") - - if self._trades_pagination == 'time': - return await self._async_get_trade_history_time( - pair=pair, since=since, until=until) - elif self._trades_pagination == 'id': - return await self._async_get_trade_history_id( - pair=pair, since=since, until=until, from_id=from_id - ) - else: - raise OperationalException(f"Exchange {self.name} does use neither time, " - f"nor id based pagination") - - def get_historic_trades(self, pair: str, - since: Optional[int] = None, - until: Optional[int] = None, - from_id: Optional[str] = None) -> Tuple[str, List]: - """ - Get trade history data using asyncio. - Handles all async work and returns the list of candles. - Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call. - :param pair: Pair to download - :param since: Timestamp in milliseconds to get history from - :param until: Timestamp in milliseconds. Defaults to current timestamp if not defined. - :param from_id: Download data starting with ID (if id is known) - :returns List of trade data - """ - if not self.exchange_has("fetchTrades"): - raise OperationalException("This exchange does not suport downloading Trades.") - - return asyncio.get_event_loop().run_until_complete( - self._async_get_trade_history(pair=pair, since=since, - until=until, from_id=from_id)) + if stoploss_order: + return self.fetch_stoploss_order(order_id, pair) + return self.fetch_order(order_id, pair) def check_order_canceled_empty(self, order: Dict) -> bool: """ @@ -1039,16 +729,18 @@ class Exchange: :param order: Order dict as returned from fetch_order() :return: True if order has been cancelled without being filled, False otherwise. """ - return order.get('status') in ('closed', 'canceled') and order.get('filled') == 0.0 + return (order.get('status') in ('closed', 'canceled', 'cancelled') + and order.get('filled') == 0.0) @retrier def cancel_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - order = self._dry_run_open_orders.get(order_id) - if order: + try: + order = self.fetch_dry_run_order(order_id) + order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) return order - else: + except InvalidOrderException: return {} try: @@ -1098,55 +790,106 @@ class Exchange: return order - @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) - def fetch_order(self, order_id: str, pair: str) -> Dict: - if self._config['dry_run']: - try: - order = self._dry_run_open_orders[order_id] - return order - except KeyError as e: - # Gracefully handle errors with dry-run orders. - raise InvalidOrderException( - f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: + """ + Cancel stoploss order returning a result. + Creates a fake result if cancel order returns a non-usable result + and fetch_order does not work (certain exchanges don't return cancelled orders) + :param order_id: stoploss-order-id to cancel + :param pair: Pair corresponding to order_id + :param amount: Amount to use for fake response + :return: Result from either cancel_order if usable, or fetch_order + """ + corder = self.cancel_stoploss_order(order_id, pair) + if self.is_cancel_order_result_suitable(corder): + return corder try: - return self._api.fetch_order(order_id, pair) - except ccxt.OrderNotFound as e: - raise RetryableOrderError( - f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e - except ccxt.InvalidOrder as e: - raise InvalidOrderException( - f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e + order = self.fetch_stoploss_order(order_id, pair) + except InvalidOrderException: + logger.warning(f"Could not fetch cancelled stoploss order {order_id}.") + order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + + return order + + @retrier + def get_balances(self) -> dict: + + try: + balances = self._api.fetch_balance() + # Remove additional info from ccxt results + balances.pop("info", None) + balances.pop("free", None) + balances.pop("total", None) + balances.pop("used", None) + + return balances except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( - f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e + f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e - # Assign method to fetch_stoploss_order to allow easy overriding in other classes - fetch_stoploss_order = fetch_order + @retrier + def get_tickers(self, cached: bool = False) -> Dict: + """ + :param cached: Allow cached result + :return: fetch_tickers result + """ + if cached: + tickers = self._fetch_tickers_cache.get('fetch_tickers') + if tickers: + return tickers + try: + tickers = self._api.fetch_tickers() + self._fetch_tickers_cache['fetch_tickers'] = tickers + return tickers + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching tickers in batch. ' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load tickers due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e - def fetch_order_or_stoploss_order(self, order_id: str, pair: str, - stoploss_order: bool = False) -> Dict: - """ - Simple wrapper calling either fetch_order or fetch_stoploss_order depending on - the stoploss_order parameter - :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. - """ - if stoploss_order: - return self.fetch_stoploss_order(order_id, pair) - return self.fetch_order(order_id, pair) + # Pricing info + + @retrier + def fetch_ticker(self, pair: str) -> dict: + try: + if (pair not in self.markets or + self.markets[pair].get('active', False) is False): + raise ExchangeError(f"Pair {pair} not available") + data = self._api.fetch_ticker(pair) + return data + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e @staticmethod - def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]]): + def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]], + range_required: bool = True): """ Get next greater value in the list. Used by fetch_l2_order_book if the api only supports a limited range """ if not limit_range: return limit - return min([x for x in limit_range if limit <= x] + [max(limit_range)]) + + result = min([x for x in limit_range if limit <= x] + [max(limit_range)]) + if not range_required and limit > result: + # Range is not required - we can use None as parameter. + return None + return result @retrier def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: @@ -1156,7 +899,8 @@ class Exchange: Returns a dict in the format {'asks': [price, volume], 'bids': [price, volume]} """ - limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range']) + limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'], + self._ft_has['l2_limit_range_required']) try: return self._api.fetch_l2_order_book(pair, limit1) @@ -1172,6 +916,103 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, + order_book_min: int = 1): + """ + Helper generator to query orderbook in loop (used for early sell-order placing) + """ + order_book = self.fetch_l2_order_book(pair, order_book_max) + for i in range(order_book_min, order_book_max + 1): + yield order_book[side][i - 1][0] + + def get_buy_rate(self, pair: str, refresh: bool) -> float: + """ + Calculates bid target between current ask price and last price + :param pair: Pair to get rate for + :param refresh: allow cached data + :return: float: Price + :raises PricingError if orderbook price could not be determined. + """ + if not refresh: + rate = self._buy_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.debug(f"Using cached buy rate for {pair}.") + return rate + + bid_strategy = self._config.get('bid_strategy', {}) + if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): + + order_book_top = bid_strategy.get('order_book_top', 1) + order_book = self.fetch_l2_order_book(pair, order_book_top) + logger.debug('order_book %s', order_book) + # top 1 = index 0 + try: + rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] + except (IndexError, KeyError) as e: + logger.warning( + "Buy Price from orderbook could not be determined." + f"Orderbook: {order_book}" + ) + raise PricingError from e + logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side " + f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}") + used_rate = rate_from_l2 + else: + logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") + ticker = self.fetch_ticker(pair) + ticker_rate = ticker[bid_strategy['price_side']] + if ticker['last'] and ticker_rate > ticker['last']: + balance = bid_strategy['ask_last_balance'] + ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) + used_rate = ticker_rate + + self._buy_rate_cache[pair] = used_rate + + return used_rate + + def get_sell_rate(self, pair: str, refresh: bool) -> float: + """ + Get sell rate - either using ticker bid or first bid based on orderbook + or remain static in any other case since it's not updating. + :param pair: Pair to get rate for + :param refresh: allow cached data + :return: Bid rate + :raises PricingError if price could not be determined. + """ + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.debug(f"Using cached sell rate for {pair}.") + return rate + + ask_strategy = self._config.get('ask_strategy', {}) + if ask_strategy.get('use_order_book', False): + # This code is only used for notifications, selling uses the generator directly + logger.info( + f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." + ) + try: + rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) + except (IndexError, KeyError) as e: + logger.warning("Sell Price at location from orderbook could not be determined.") + raise PricingError from e + else: + ticker = self.fetch_ticker(pair) + ticker_rate = ticker[ask_strategy['price_side']] + if ticker['last'] and ticker_rate < ticker['last']: + balance = ask_strategy.get('bid_last_balance', 0.0) + ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last']) + rate = ticker_rate + + if rate is None: + raise PricingError(f"Sell-Rate for {pair} was empty.") + self._sell_rate_cache[pair] = rate + return rate + + # Fee handling + @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: """ @@ -1210,10 +1051,15 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def get_order_id_conditional(self, order: Dict[str, Any]) -> str: + return order['id'] + @retrier def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, price: float = 1, taker_or_maker: str = 'maker') -> float: try: + if self._config['dry_run'] and self._config.get('fee', None) is not None: + return self._config['fee'] # validate that markets are loaded before trying to get fee if self._api.markets is None or len(self._api.markets) == 0: self._api.load_markets() @@ -1285,13 +1131,334 @@ class Exchange: order['fee']['currency'], self.calculate_fee_rate(order)) + # Historic data -def is_exchange_bad(exchange_name: str) -> bool: - return exchange_name in BAD_EXCHANGES + def get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int) -> List: + """ + Get candle history using asyncio and returns the list of candles. + Handles all async work for this. + Async over one pair, assuming we get `self.ohlcv_candle_limit()` candles per call. + :param pair: Pair to download + :param timeframe: Timeframe to get data for + :param since_ms: Timestamp in milliseconds to get history from + :return: List with candle (OHLCV) data + """ + return asyncio.get_event_loop().run_until_complete( + self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, + since_ms=since_ms)) + def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, + since_ms: int) -> DataFrame: + """ + Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe + :param pair: Pair to download + :param timeframe: Timeframe to get data for + :param since_ms: Timestamp in milliseconds to get history from + :return: OHLCV DataFrame + """ + ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) + return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) -def get_exchange_bad_reason(exchange_name: str) -> str: - return BAD_EXCHANGES.get(exchange_name, "") + async def _async_get_historic_ohlcv(self, pair: str, + timeframe: str, + since_ms: int) -> List: + """ + Download historic ohlcv + """ + + one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) + logger.debug( + "one_call: %s msecs (%s)", + one_call, + arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) + ) + input_coroutines = [self._async_get_candle_history( + pair, timeframe, since) for since in + range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] + + results = await asyncio.gather(*input_coroutines, return_exceptions=True) + + # Combine gathered results + data: List = [] + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) + # Sort data again after extending the result - above calls return in "async order" + data = sorted(data, key=lambda x: x[0]) + logger.info("Downloaded data for %s with length %s.", pair, len(data)) + return data + + def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, + since_ms: Optional[int] = None, cache: bool = True + ) -> Dict[Tuple[str, str], DataFrame]: + """ + Refresh in-memory OHLCV asynchronously and set `_klines` with the result + Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). + Only used in the dataprovider.refresh() method. + :param pair_list: List of 2 element tuples containing pair, interval to refresh + :param since_ms: time since when to download, in milliseconds + :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists + :return: Dict of [{(pair, timeframe): Dataframe}] + """ + logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) + + input_coroutines = [] + + # Gather coroutines to run + for pair, timeframe in set(pair_list): + if (((pair, timeframe) not in self._klines) + or self._now_is_time_to_refresh(pair, timeframe)): + input_coroutines.append(self._async_get_candle_history(pair, timeframe, + since_ms=since_ms)) + else: + logger.debug( + "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", + pair, timeframe + ) + + results = asyncio.get_event_loop().run_until_complete( + asyncio.gather(*input_coroutines, return_exceptions=True)) + + results_df = {} + # handle caching + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple (has 3 elements) + pair, timeframe, ticks = res + # keeping last candle time as last refreshed time of the pair + if ticks: + self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 + # keeping parsed dataframe in cache + ohlcv_df = ohlcv_to_dataframe( + ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + results_df[(pair, timeframe)] = ohlcv_df + if cache: + self._klines[(pair, timeframe)] = ohlcv_df + return results_df + + def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool: + # Timeframe in seconds + interval_in_sec = timeframe_to_seconds(timeframe) + + return not ((self._pairs_last_refresh_time.get((pair, timeframe), 0) + + interval_in_sec) >= arrow.utcnow().int_timestamp) + + @retrier_async + async def _async_get_candle_history(self, pair: str, timeframe: str, + since_ms: Optional[int] = None) -> Tuple[str, str, List]: + """ + Asynchronously get candle history data using fetch_ohlcv + returns tuple: (pair, timeframe, ohlcv_list) + """ + try: + # Fetch OHLCV asynchronously + s = '(' + arrow.get(since_ms // 1000).isoformat() + ') ' if since_ms is not None else '' + logger.debug( + "Fetching pair %s, interval %s, since %s %s...", + pair, timeframe, since_ms, s + ) + params = self._ft_has.get('ohlcv_params', {}) + data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, + since=since_ms, + limit=self.ohlcv_candle_limit(timeframe), + params=params) + + # Some exchanges sort OHLCV in ASC order and others in DESC. + # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) + # while GDAX returns the list of OHLCV in DESC order (newest first, oldest last) + # Only sort if necessary to save computing time + try: + if data and data[0][0] > data[-1][0]: + data = sorted(data, key=lambda x: x[0]) + except IndexError: + logger.exception("Error loading %s. Result was %s.", pair, data) + return pair, timeframe, [] + logger.debug("Done fetching pair %s, interval %s ...", pair, timeframe) + return pair, timeframe, data + + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching historical ' + f'candle (OHLCV) data. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError(f'Could not fetch historical candle (OHLCV) data ' + f'for pair {pair} due to {e.__class__.__name__}. ' + f'Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(f'Could not fetch historical candle (OHLCV) data ' + f'for pair {pair}. Message: {e}') from e + + # Fetch historic trades + + @retrier_async + async def _async_fetch_trades(self, pair: str, + since: Optional[int] = None, + params: Optional[dict] = None) -> List[List]: + """ + Asyncronously gets trade history using fetch_trades. + Handles exchange errors, does one call to the exchange. + :param pair: Pair to fetch trade data for + :param since: Since as integer timestamp in milliseconds + returns: List of dicts containing trades + """ + try: + # fetch trades asynchronously + if params: + logger.debug("Fetching trades for pair %s, params: %s ", pair, params) + trades = await self._api_async.fetch_trades(pair, params=params, limit=1000) + else: + logger.debug( + "Fetching trades for pair %s, since %s %s...", + pair, since, + '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else '' + ) + trades = await self._api_async.fetch_trades(pair, since=since, limit=1000) + return trades_dict_to_list(trades) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching historical trade data.' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError(f'Could not load trade history due to {e.__class__.__name__}. ' + f'Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(f'Could not fetch trade data. Msg: {e}') from e + + async def _async_get_trade_history_id(self, pair: str, + until: int, + since: Optional[int] = None, + from_id: Optional[str] = None) -> Tuple[str, List[List]]: + """ + Asyncronously gets trade history using fetch_trades + use this when exchange uses id-based iteration (check `self._trades_pagination`) + :param pair: Pair to fetch trade data for + :param since: Since as integer timestamp in milliseconds + :param until: Until as integer timestamp in milliseconds + :param from_id: Download data starting with ID (if id is known). Ignores "since" if set. + returns tuple: (pair, trades-list) + """ + + trades: List[List] = [] + + if not from_id: + # Fetch first elements using timebased method to get an ID to paginate on + # Depending on the Exchange, this can introduce a drift at the start of the interval + # of up to an hour. + # e.g. Binance returns the "last 1000" candles within a 1h time interval + # - so we will miss the first trades. + t = await self._async_fetch_trades(pair, since=since) + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id + from_id = t[-1][1] + trades.extend(t[:-1]) + while True: + t = await self._async_fetch_trades(pair, + params={self._trades_pagination_arg: from_id}) + if t: + # Skip last id since its the key for the next call + trades.extend(t[:-1]) + if from_id == t[-1][1] or t[-1][0] > until: + logger.debug(f"Stopping because from_id did not change. " + f"Reached {t[-1][0]} > {until}") + # Reached the end of the defined-download period - add last trade as well. + trades.extend(t[-1:]) + break + + from_id = t[-1][1] + else: + break + + return (pair, trades) + + async def _async_get_trade_history_time(self, pair: str, until: int, + since: Optional[int] = None) -> Tuple[str, List[List]]: + """ + Asyncronously gets trade history using fetch_trades, + when the exchange uses time-based iteration (check `self._trades_pagination`) + :param pair: Pair to fetch trade data for + :param since: Since as integer timestamp in milliseconds + :param until: Until as integer timestamp in milliseconds + returns tuple: (pair, trades-list) + """ + + trades: List[List] = [] + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id + while True: + t = await self._async_fetch_trades(pair, since=since) + if t: + since = t[-1][0] + trades.extend(t) + # Reached the end of the defined-download period + if until and t[-1][0] > until: + logger.debug( + f"Stopping because until was reached. {t[-1][0]} > {until}") + break + else: + break + + return (pair, trades) + + async def _async_get_trade_history(self, pair: str, + since: Optional[int] = None, + until: Optional[int] = None, + from_id: Optional[str] = None) -> Tuple[str, List[List]]: + """ + Async wrapper handling downloading trades using either time or id based methods. + """ + + logger.debug(f"_async_get_trade_history(), pair: {pair}, " + f"since: {since}, until: {until}, from_id: {from_id}") + + if until is None: + until = ccxt.Exchange.milliseconds() + logger.debug(f"Exchange milliseconds: {until}") + + if self._trades_pagination == 'time': + return await self._async_get_trade_history_time( + pair=pair, since=since, until=until) + elif self._trades_pagination == 'id': + return await self._async_get_trade_history_id( + pair=pair, since=since, until=until, from_id=from_id + ) + else: + raise OperationalException(f"Exchange {self.name} does use neither time, " + f"nor id based pagination") + + def get_historic_trades(self, pair: str, + since: Optional[int] = None, + until: Optional[int] = None, + from_id: Optional[str] = None) -> Tuple[str, List]: + """ + Get trade history data using asyncio. + Handles all async work and returns the list of candles. + Async over one pair, assuming we get `self.ohlcv_candle_limit()` candles per call. + :param pair: Pair to download + :param since: Timestamp in milliseconds to get history from + :param until: Timestamp in milliseconds. Defaults to current timestamp if not defined. + :param from_id: Download data starting with ID (if id is known) + :returns List of trade data + """ + if not self.exchange_has("fetchTrades"): + raise OperationalException("This exchange does not suport downloading Trades.") + + return asyncio.get_event_loop().run_until_complete( + self._async_get_trade_history(pair=pair, since=since, + until=until, from_id=from_id)) def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool: @@ -1314,7 +1481,36 @@ def available_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: Return exchanges available to the bot, i.e. non-bad exchanges in the ccxt list """ exchanges = ccxt_exchanges(ccxt_module) - return [x for x in exchanges if not is_exchange_bad(x)] + return [x for x in exchanges if validate_exchange(x)[0]] + + +def validate_exchange(exchange: str) -> Tuple[bool, str]: + ex_mod = getattr(ccxt, exchange.lower())() + if not ex_mod or not ex_mod.has: + return False, '' + missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True] + if missing: + return False, f"missing: {', '.join(missing)}" + + missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)] + + if exchange.lower() in BAD_EXCHANGES: + return False, BAD_EXCHANGES.get(exchange.lower(), '') + if missing_opt: + return True, f"missing opt: {', '.join(missing_opt)}" + + return True, '' + + +def validate_exchanges(all_exchanges: bool) -> List[Tuple[str, bool, str]]: + """ + :return: List of tuples with exchangename, valid, reason. + """ + exchanges = ccxt_exchanges() if all_exchanges else available_exchanges() + exchanges_valid = [ + (e, *validate_exchange(e)) for e in exchanges + ] + return exchanges_valid def timeframe_to_seconds(timeframe: str) -> int: diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index f05490cbb..3184c2524 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -8,6 +8,7 @@ from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, Invali OperationalException, TemporaryError) from freqtrade.exchange import Exchange from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier +from freqtrade.misc import safe_value_fallback2 logger = logging.getLogger(__name__) @@ -53,7 +54,7 @@ class Ftx(Exchange): stop_price = self.price_to_precision(pair, stop_price) if self._config['dry_run']: - dry_order = self.dry_run_order( + dry_order = self.create_dry_run_order( pair, ordertype, "sell", amount, stop_price) return dry_order @@ -63,10 +64,11 @@ class Ftx(Exchange): # set orderPrice to place limit order, otherwise it's a market order params['orderPrice'] = limit_rate + params['stopPrice'] = stop_price amount = self.amount_to_precision(pair, amount) order = self._api.create_order(symbol=pair, type=ordertype, side='sell', - amount=amount, price=stop_price, params=params) + amount=amount, params=params) logger.info('stoploss order added for %s. ' 'stop price: %s.', pair, stop_price) return order @@ -91,18 +93,24 @@ class Ftx(Exchange): @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - try: - order = self._dry_run_open_orders[order_id] - return order - except KeyError as e: - # Gracefully handle errors with dry-run orders. - raise InvalidOrderException( - f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + return self.fetch_dry_run_order(order_id) + try: orders = self._api.fetch_orders(pair, None, params={'type': 'stop'}) order = [order for order in orders if order['id'] == order_id] if len(order) == 1: + if order[0].get('status') == 'closed': + # Trigger order was triggered ... + real_order_id = order[0].get('info', {}).get('orderId') + + order1 = self._api.fetch_order(real_order_id, pair) + # Fake type to stop - as this was really a stop order. + order1['id_stop'] = order1['id'] + order1['id'] = order_id + order1['type'] = 'stop' + order1['status_stop'] = 'triggered' + return order1 return order[0] else: raise InvalidOrderException(f"Could not get stoploss order for id {order_id}") @@ -134,3 +142,8 @@ class Ftx(Exchange): f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + + def get_order_id_conditional(self, order: Dict[str, Any]) -> str: + if order['type'] == 'stop': + return safe_value_fallback2(order, order, 'id_stop', 'id') + return order['id'] diff --git a/freqtrade/exchange/hitbtc.py b/freqtrade/exchange/hitbtc.py new file mode 100644 index 000000000..a48c9a198 --- /dev/null +++ b/freqtrade/exchange/hitbtc.py @@ -0,0 +1,23 @@ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Hitbtc(Exchange): + """ + Hitbtc exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + _ft_has: Dict = { + "ohlcv_candle_limit": 1000, + "ohlcv_params": {"sort": "DESC"} + } diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 724b11189..6f1fa409a 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -53,6 +53,8 @@ class Kraken(Exchange): # x["side"], x["amount"], ) for x in orders] for bal in balances: + if not isinstance(balances[bal], dict): + continue balances[bal]['used'] = sum(order[1] for order in order_list if order[0] == bal) balances[bal]['free'] = balances[bal]['total'] - balances[bal]['used'] @@ -92,7 +94,7 @@ class Kraken(Exchange): stop_price = self.price_to_precision(pair, stop_price) if self._config['dry_run']: - dry_order = self.dry_run_order( + dry_order = self.create_dry_run_order( pair, ordertype, "sell", amount, stop_price) return dry_order diff --git a/freqtrade/exchange/kucoin.py b/freqtrade/exchange/kucoin.py new file mode 100644 index 000000000..22886a1d8 --- /dev/null +++ b/freqtrade/exchange/kucoin.py @@ -0,0 +1,24 @@ +""" Kucoin exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Kucoin(Exchange): + """ + Kucoin exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + _ft_has: Dict = { + "l2_limit_range": [20, 100], + "l2_limit_range_required": False, + } diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index a6eb75d5b..a2e7fcb5d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -10,13 +10,13 @@ from threading import Lock from typing import Any, Dict, List, Optional import arrow -from cachetools import TTLCache from freqtrade import __version__, constants from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge +from freqtrade.enums import RPCMessageType, SellType, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds @@ -26,9 +26,8 @@ from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.state import State -from freqtrade.strategy.interface import IStrategy, SellType +from freqtrade.rpc import RPCManager +from freqtrade.strategy.interface import IStrategy, SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -48,6 +47,7 @@ class FreqtradeBot(LoggingMixin): :param config: configuration dict, you can use Configuration.get_config() to get the config dict. """ + self.active_pair_whitelist: List[str] = [] logger.info('Starting freqtrade %s', __version__) @@ -57,12 +57,6 @@ class FreqtradeBot(LoggingMixin): # Init objects self.config = config - # Cache values for 1800 to avoid frequent polling of the exchange for prices - # Caching only applies to RPC methods, so prices for open trades are still - # refreshed once every iteration. - self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) - self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) - self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) # Check config consistency here since strategies can set certain options @@ -76,12 +70,19 @@ class FreqtradeBot(LoggingMixin): PairLocks.timeframe = self.config['timeframe'] + self.protections = ProtectionManager(self.config) + + # RPC runs in separate threads, can start handling external commands just after + # initialization, even before Freqtradebot has a chance to start its throttling, + # so anything in the Freqtradebot instance should be ready (initialized), including + # the initial state of the bot. + # Keep this at the end of this initialization method. + self.rpc: RPCManager = RPCManager(self) + self.pairlists = PairListManager(self.exchange, self.config) self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) - self.protections = ProtectionManager(self.config) - # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass @@ -97,12 +98,6 @@ class FreqtradeBot(LoggingMixin): initial_state = self.config.get('initial_state') self.state = State[initial_state.upper()] if initial_state else State.STOPPED - # RPC runs in separate threads, can start handling external commands just after - # initialization, even before Freqtradebot has a chance to start its throttling, - # so anything in the Freqtradebot instance should be ready (initialized), including - # the initial state of the bot. - # Keep this at the end of this initialization method. - self.rpc: RPCManager = RPCManager(self) # Protect sell-logic from forcesell and viceversa self._sell_lock = Lock() LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) @@ -113,7 +108,7 @@ class FreqtradeBot(LoggingMixin): via RPC about changes in the bot status. """ self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, + 'type': RPCMessageType.STATUS, 'status': msg }) @@ -179,6 +174,7 @@ class FreqtradeBot(LoggingMixin): # Without this, freqtrade my try to recreate stoploss_on_exchange orders # while selling is in process, since telegram messages arrive in an different thread. with self._sell_lock: + trades = Trade.get_open_trades() # First process current opened trades (positions) self.exit_positions(trades) @@ -186,7 +182,7 @@ class FreqtradeBot(LoggingMixin): if self.get_free_open_trades(): self.enter_positions() - Trade.session.flush() + Trade.commit() def process_stopped(self) -> None: """ @@ -204,7 +200,7 @@ class FreqtradeBot(LoggingMixin): if len(open_trades) != 0: msg = { - 'type': RPCMessageType.WARNING_NOTIFICATION, + 'type': RPCMessageType.WARNING, 'status': f"{len(open_trades)} open trades active.\n\n" f"Handle these trades manually on {self.exchange.name}, " f"or '/start' the bot again and use '/stopbuy' " @@ -224,7 +220,7 @@ class FreqtradeBot(LoggingMixin): # Calculating Edge positioning if self.edge: - self.edge.calculate() + self.edge.calculate(_whitelist) _whitelist = self.edge.adjust(_whitelist) if trades: @@ -266,7 +262,7 @@ class FreqtradeBot(LoggingMixin): def update_closed_trades_without_assigned_fees(self): """ Update closed trades without close fees assigned. - Only acts when Orders are in the database, otherwise the last orderid is unknown. + Only acts when Orders are in the database, otherwise the last order-id is unknown. """ if self.config['dry_run']: # Updating open orders in dry-run does not make sense and will fail. @@ -341,7 +337,7 @@ class FreqtradeBot(LoggingMixin): # Assume this as the open order trade.open_order_id = order.order_id if fo: - logger.info(f"Found {order} for trade {trade}.jj") + logger.info(f"Found {order} for trade {trade}.") self.update_trade_state(trade, order.order_id, fo, stoploss_order=order.ft_order_side == 'stoploss') @@ -377,7 +373,7 @@ class FreqtradeBot(LoggingMixin): if lock: self.log_once(f"Global pairlock active until " f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}. " - "Not creating new trades.", logger.info) + f"Not creating new trades, reason: {lock.reason}.", logger.info) else: self.log_once("Global pairlock active. Not creating new trades.", logger.info) return trades_created @@ -393,52 +389,6 @@ class FreqtradeBot(LoggingMixin): return trades_created - def get_buy_rate(self, pair: str, refresh: bool) -> float: - """ - Calculates bid target between current ask price and last price - :param pair: Pair to get rate for - :param refresh: allow cached data - :return: float: Price - """ - if not refresh: - rate = self._buy_rate_cache.get(pair) - # Check if cache has been invalidated - if rate: - logger.debug(f"Using cached buy rate for {pair}.") - return rate - - bid_strategy = self.config.get('bid_strategy', {}) - if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): - logger.info( - f"Getting price from order book {bid_strategy['price_side'].capitalize()} side." - ) - order_book_top = bid_strategy.get('order_book_top', 1) - order_book = self.exchange.fetch_l2_order_book(pair, order_book_top) - logger.debug('order_book %s', order_book) - # top 1 = index 0 - try: - rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] - except (IndexError, KeyError) as e: - logger.warning( - "Buy Price from orderbook could not be determined." - f"Orderbook: {order_book}" - ) - raise PricingError from e - logger.info(f'...top {order_book_top} order book buy rate {rate_from_l2:.8f}') - used_rate = rate_from_l2 - else: - logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") - ticker = self.exchange.fetch_ticker(pair) - ticker_rate = ticker[bid_strategy['price_side']] - if ticker['last'] and ticker_rate > ticker['last']: - balance = self.config['bid_strategy']['ask_last_balance'] - ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) - used_rate = ticker_rate - - self._buy_rate_cache[pair] = used_rate - - return used_rate - def create_trade(self, pair: str) -> bool: """ Check the implemented trading strategy for buy signals. @@ -456,7 +406,8 @@ class FreqtradeBot(LoggingMixin): lock = PairLocks.get_pair_longest_lock(pair, nowtime) if lock: self.log_once(f"Pair {pair} is still locked until " - f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}.", + f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} " + f"due to {lock.reason}.", logger.info) else: self.log_once(f"Pair {pair} is still locked.", logger.info) @@ -472,25 +423,22 @@ class FreqtradeBot(LoggingMixin): (buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df) if buy and not sell: - stake_amount = self.wallets.get_trade_stake_amount(pair, self.get_free_open_trades(), - self.edge) + stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge) if not stake_amount: logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.") return False - logger.info(f"Buy signal found: about create a new trade with stake_amount: " + logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: " f"{stake_amount} ...") bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {}) if ((bid_check_dom.get('enabled', False)) and (bid_check_dom.get('bids_to_ask_delta', 0) > 0)): if self._check_depth_of_market_buy(pair, bid_check_dom): - logger.info(f'Executing Buy for {pair}.') return self.execute_buy(pair, stake_amount) else: return False - logger.info(f'Executing Buy for {pair}') return self.execute_buy(pair, stake_amount) else: return False @@ -519,7 +467,8 @@ class FreqtradeBot(LoggingMixin): logger.info(f"Bids to asks delta for {pair} does not satisfy condition.") return False - def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None) -> bool: + def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None, + forcebuy: bool = False) -> bool: """ Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY @@ -531,7 +480,7 @@ class FreqtradeBot(LoggingMixin): buy_limit_requested = price else: # Calculate price - buy_limit_requested = self.get_buy_rate(pair, True) + buy_limit_requested = self.exchange.get_buy_rate(pair, True) if not buy_limit_requested: raise PricingError('Could not determine buy price.') @@ -547,9 +496,13 @@ class FreqtradeBot(LoggingMixin): amount = stake_amount / buy_limit_requested order_type = self.strategy.order_types['buy'] + if forcebuy: + # Forcebuy can define a different ordertype + order_type = self.strategy.order_types.get('forcebuy', order_type) + if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested, - time_in_force=time_in_force): + time_in_force=time_in_force, current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of buying {pair}") return False amount = self.exchange.amount_to_precision(pair, amount) @@ -598,6 +551,7 @@ class FreqtradeBot(LoggingMixin): pair=pair, stake_amount=stake_amount, amount=amount, + is_open=True, amount_requested=amount_requested, fee_open=fee, fee_close=fee, @@ -615,8 +569,8 @@ class FreqtradeBot(LoggingMixin): if order_status == 'closed': self.update_trade_state(trade, order_id, order) - Trade.session.add(trade) - Trade.session.flush() + Trade.query.session.add(trade) + Trade.commit() # Updating wallets self.wallets.update() @@ -627,11 +581,11 @@ class FreqtradeBot(LoggingMixin): def _notify_buy(self, trade: Trade, order_type: str) -> None: """ - Sends rpc notification when a buy occured. + Sends rpc notification when a buy occurred. """ msg = { 'trade_id': trade.id, - 'type': RPCMessageType.BUY_NOTIFICATION, + 'type': RPCMessageType.BUY, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, 'limit': trade.open_rate, @@ -649,13 +603,13 @@ class FreqtradeBot(LoggingMixin): def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ - Sends rpc notification when a buy cancel occured. + Sends rpc notification when a buy cancel occurred. """ - current_rate = self.get_buy_rate(trade.pair, False) + current_rate = self.exchange.get_buy_rate(trade.pair, False) msg = { 'trade_id': trade.id, - 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, + 'type': RPCMessageType.BUY_CANCEL, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, 'limit': trade.open_rate, @@ -672,6 +626,21 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) + def _notify_buy_fill(self, trade: Trade) -> None: + msg = { + 'trade_id': trade.id, + 'type': RPCMessageType.BUY_FILL, + 'exchange': self.exchange.name.capitalize(), + 'pair': trade.pair, + 'open_rate': trade.open_rate, + 'stake_amount': trade.stake_amount, + 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), + 'amount': trade.amount, + 'open_date': trade.open_date, + } + self.rpc.send_msg(msg) + # # SELL / exit positions / close trades logic and methods # @@ -687,6 +656,7 @@ class FreqtradeBot(LoggingMixin): if (self.strategy.order_types.get('stoploss_on_exchange') and self.handle_stoploss_on_exchange(trade)): trades_closed += 1 + Trade.commit() continue # Check if we can sell our current pair if trade.open_order_id is None and trade.is_open and self.handle_trade(trade): @@ -695,56 +665,12 @@ class FreqtradeBot(LoggingMixin): except DependencyException as exception: logger.warning('Unable to sell trade %s: %s', trade.pair, exception) - # Updating wallets if any trade occured + # Updating wallets if any trade occurred if trades_closed: self.wallets.update() return trades_closed - def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, - order_book_min: int = 1): - """ - Helper generator to query orderbook in loop (used for early sell-order placing) - """ - order_book = self.exchange.fetch_l2_order_book(pair, order_book_max) - for i in range(order_book_min, order_book_max + 1): - yield order_book[side][i - 1][0] - - def get_sell_rate(self, pair: str, refresh: bool) -> float: - """ - Get sell rate - either using ticker bid or first bid based on orderbook - The orderbook portion is only used for rpc messaging, which would otherwise fail - for BitMex (has no bid/ask in fetch_ticker) - or remain static in any other case since it's not updating. - :param pair: Pair to get rate for - :param refresh: allow cached data - :return: Bid rate - """ - if not refresh: - rate = self._sell_rate_cache.get(pair) - # Check if cache has been invalidated - if rate: - logger.debug(f"Using cached sell rate for {pair}.") - return rate - - ask_strategy = self.config.get('ask_strategy', {}) - if ask_strategy.get('use_order_book', False): - # This code is only used for notifications, selling uses the generator directly - logger.info( - f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." - ) - try: - rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) - except (IndexError, KeyError) as e: - logger.warning("Sell Price at location from orderbook could not be determined.") - raise PricingError from e - else: - rate = self.exchange.fetch_ticker(pair)[ask_strategy['price_side']] - if rate is None: - raise PricingError(f"Sell-Rate for {pair} was empty.") - self._sell_rate_cache[pair] = rate - return rate - def handle_trade(self, trade: Trade) -> bool: """ Sells the current pair if the threshold is reached and updates the trade record. @@ -772,9 +698,9 @@ class FreqtradeBot(LoggingMixin): logger.debug(f'Using order book between {order_book_min} and {order_book_max} ' f'for selling {trade.pair}...') - order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s", - order_book_min=order_book_min, - order_book_max=order_book_max) + order_book = self.exchange._order_book_gen( + trade.pair, f"{config_ask_strategy['price_side']}s", + order_book_min=order_book_min, order_book_max=order_book_max) for i in range(order_book_min, order_book_max + 1): try: sell_rate = next(order_book) @@ -787,14 +713,14 @@ class FreqtradeBot(LoggingMixin): f"{sell_rate:0.8f}") # Assign sell-rate to cache - otherwise sell-rate is never updated in the cache, # resulting in outdated RPC messages - self._sell_rate_cache[trade.pair] = sell_rate + self.exchange._sell_rate_cache[trade.pair] = sell_rate if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True else: logger.debug('checking sell') - sell_rate = self.get_sell_rate(trade.pair, True) + sell_rate = self.exchange.get_sell_rate(trade.pair, True) if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True @@ -826,7 +752,8 @@ class FreqtradeBot(LoggingMixin): trade.stoploss_order_id = None logger.error(f'Unable to place a stoploss order on exchange. {e}') logger.warning('Selling the trade forcefully') - self.execute_sell(trade, trade.stop_loss, sell_reason=SellType.EMERGENCY_SELL) + self.execute_sell(trade, trade.stop_loss, sell_reason=SellCheckTuple( + sell_type=SellType.EMERGENCY_SELL)) except ExchangeError: trade.stoploss_order_id = None @@ -889,8 +816,13 @@ class FreqtradeBot(LoggingMixin): logger.warning('Stoploss order was cancelled, but unable to recreate one.') # Finally we check if stoploss on exchange should be moved up because of trailing. - if stoploss_order and (self.config.get('trailing_stop', False) - or self.config.get('use_custom_stoploss', False)): + # Triggered Orders are now real orders - so don't replace stoploss anymore + if ( + stoploss_order + and stoploss_order.get('status_stop') != 'triggered' + and (self.config.get('trailing_stop', False) + or self.config.get('use_custom_stoploss', False)) + ): # if trailing stoploss is enabled we check if stoploss value has changed # in which case we cancel stoploss order and put another one with new # value immediately @@ -907,14 +839,15 @@ class FreqtradeBot(LoggingMixin): :return: None """ if self.exchange.stoploss_adjust(trade.stop_loss, order): - # we check if the update is neccesary + # we check if the update is necessary update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: # cancelling the current stoploss on exchange first logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} " f"(orderid:{order['id']}) in order to add another one ...") try: - co = self.exchange.cancel_stoploss_order(order['id'], trade.pair) + co = self.exchange.cancel_stoploss_order_with_result(order['id'], trade.pair, + trade.amount) trade.update_order(co) except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {order['id']} " @@ -931,13 +864,13 @@ class FreqtradeBot(LoggingMixin): Check and execute sell """ should_sell = self.strategy.should_sell( - trade, sell_rate, datetime.utcnow(), buy, sell, + trade, sell_rate, datetime.now(timezone.utc), buy, sell, force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 ) if should_sell.sell_flag: logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}') - self.execute_sell(trade, sell_rate, should_sell.sell_type) + self.execute_sell(trade, sell_rate, should_sell) return True return False @@ -948,15 +881,16 @@ class FreqtradeBot(LoggingMixin): timeout = self.config.get('unfilledtimeout', {}).get(side) ordertime = arrow.get(order['datetime']).datetime if timeout is not None: - timeout_threshold = arrow.utcnow().shift(minutes=-timeout).datetime - + timeout_unit = self.config.get('unfilledtimeout', {}).get('unit', 'minutes') + timeout_kwargs = {timeout_unit: -timeout} + timeout_threshold = arrow.utcnow().shift(**timeout_kwargs).datetime return (order['status'] == 'open' and order['side'] == side and ordertime < timeout_threshold) return False def check_handle_timedout(self) -> None: """ - Check if any orders are timed out and cancel if neccessary + Check if any orders are timed out and cancel if necessary :param timeoutvalue: Number of minutes until order is considered timed out :return: None """ @@ -1008,6 +942,7 @@ class FreqtradeBot(LoggingMixin): elif order['side'] == 'sell': self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + Trade.commit() def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool: """ @@ -1017,13 +952,23 @@ class FreqtradeBot(LoggingMixin): was_trade_fully_canceled = False # Cancelled orders may have the status of 'canceled' or 'closed' - if order['status'] not in ('canceled', 'closed'): + if order['status'] not in ('cancelled', 'canceled', 'closed'): + filled_val = order.get('filled', 0.0) or 0.0 + filled_stake = filled_val * trade.open_rate + minstake = self.exchange.get_min_pair_stake_amount( + trade.pair, trade.open_rate, self.strategy.stoploss) + + if filled_val > 0 and filled_stake < minstake: + logger.warning( + f"Order {trade.open_order_id} for {trade.pair} not cancelled, " + f"as the filled amount of {filled_val} would result in an unsellable trade.") + return False corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair, trade.amount) # Avoid race condition where the order could not be cancelled coz its already filled. # Simply bailing here is the only safe way - as this order will then be # handled in the next iteration. - if corder.get('status') not in ('canceled', 'closed'): + if corder.get('status') not in ('cancelled', 'canceled', 'closed'): logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.") return False else: @@ -1126,16 +1071,16 @@ class FreqtradeBot(LoggingMixin): raise DependencyException( f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}") - def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> bool: + def execute_sell(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool: """ Executes a limit sell for the given trade and limit :param trade: Trade instance :param limit: limit rate for the sell order - :param sellreason: Reason the sell was triggered + :param sell_reason: Reason the sell was triggered :return: True if it succeeds (supported) False (not supported) """ sell_type = 'sell' - if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): + if sell_reason.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' # if stoploss is on exchange and we are on dry_run mode, @@ -1147,22 +1092,28 @@ class FreqtradeBot(LoggingMixin): # First cancelling stoploss on exchange ... if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id: try: - self.exchange.cancel_stoploss_order(trade.stoploss_order_id, trade.pair) + co = self.exchange.cancel_stoploss_order_with_result(trade.stoploss_order_id, + trade.pair, trade.amount) + trade.update_order(co) except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}") order_type = self.strategy.order_types[sell_type] - if sell_reason == SellType.EMERGENCY_SELL: + if sell_reason.sell_type == SellType.EMERGENCY_SELL: # Emergency sells (default to market!) order_type = self.strategy.order_types.get("emergencysell", "market") + if sell_reason.sell_type == SellType.FORCE_SELL: + # Force sells (default to the sell_type defined in the strategy, + # but we allow this value to be changed) + order_type = self.strategy.order_types.get("forcesell", order_type) amount = self._safe_sell_amount(trade.pair, trade.amount) time_in_force = self.strategy.order_time_in_force['sell'] if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, - time_in_force=time_in_force, - sell_reason=sell_reason.value): + time_in_force=time_in_force, sell_reason=sell_reason.sell_reason, + current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of selling {trade.pair}") return False @@ -1183,14 +1134,15 @@ class FreqtradeBot(LoggingMixin): trade.orders.append(order_obj) trade.open_order_id = order['id'] + trade.sell_order_status = '' trade.close_rate_requested = limit - trade.sell_reason = sell_reason.value + trade.sell_reason = sell_reason.sell_reason # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': self.update_trade_state(trade, trade.open_order_id, order) - Trade.session.flush() + Trade.commit() - # Lock pair for one candle to prevent immediate rebuys + # Lock pair for one candle to prevent immediate re-buys self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') @@ -1198,19 +1150,20 @@ class FreqtradeBot(LoggingMixin): return True - def _notify_sell(self, trade: Trade, order_type: str) -> None: + def _notify_sell(self, trade: Trade, order_type: str, fill: bool = False) -> None: """ - Sends rpc notification when a sell occured. + Sends rpc notification when a sell occurred. """ profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) # Use cached rates here - it was updated seconds ago. - current_rate = self.get_sell_rate(trade.pair, False) + current_rate = self.exchange.get_sell_rate(trade.pair, False) if not fill else None profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" msg = { - 'type': RPCMessageType.SELL_NOTIFICATION, + 'type': (RPCMessageType.SELL_FILL if fill + else RPCMessageType.SELL), 'trade_id': trade.id, 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, @@ -1219,6 +1172,7 @@ class FreqtradeBot(LoggingMixin): 'order_type': order_type, 'amount': trade.amount, 'open_rate': trade.open_rate, + 'close_rate': trade.close_rate, 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_ratio': profit_ratio, @@ -1239,7 +1193,7 @@ class FreqtradeBot(LoggingMixin): def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ - Sends rpc notification when a sell cancel occured. + Sends rpc notification when a sell cancel occurred. """ if trade.sell_order_status == reason: return @@ -1248,12 +1202,12 @@ class FreqtradeBot(LoggingMixin): profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - current_rate = self.get_sell_rate(trade.pair, False) + current_rate = self.exchange.get_sell_rate(trade.pair, False) profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" msg = { - 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'type': RPCMessageType.SELL_CANCEL, 'trade_id': trade.id, 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, @@ -1292,7 +1246,7 @@ class FreqtradeBot(LoggingMixin): Handles closing both buy and sell orders. :param trade: Trade object of the trade we're analyzing :param order_id: Order-id of the order we're analyzing - :param action_order: Already aquired order object + :param action_order: Already acquired order object :return: True if order has been cancelled without being filled partially, False otherwise """ if not order_id: @@ -1327,12 +1281,19 @@ class FreqtradeBot(LoggingMixin): # Handling of this will happen in check_handle_timeout. return True trade.update(order) + Trade.commit() # Updating wallets when order is closed if not trade.is_open: + if not stoploss_order and not trade.open_order_id: + self._notify_sell(trade, '', True) self.protections.stop_per_pair(trade.pair) self.protections.global_stop() self.wallets.update() + elif not trade.open_order_id: + # Buy fill + self._notify_buy_fill(trade) + return False def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, @@ -1356,7 +1317,7 @@ class FreqtradeBot(LoggingMixin): def get_real_amount(self, trade: Trade, order: Dict) -> float: """ Detect and update trade fee. - Calls trade.update_fee() uppon correct detection. + Calls trade.update_fee() upon correct detection. Returns modified amount if the fee was taken from the destination currency. Necessary for exchanges which charge fees in base currency (e.g. binance) :return: identical (or new) amount for the trade @@ -1389,8 +1350,8 @@ class FreqtradeBot(LoggingMixin): """ fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee. """ - trades = self.exchange.get_trades_for_order(order['id'], trade.pair, - trade.open_date) + trades = self.exchange.get_trades_for_order(self.exchange.get_order_id_conditional(order), + trade.pair, trade.open_date) if len(trades) == 0: logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 7bbc24056..2e255901e 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -6,7 +6,7 @@ import logging import re from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, Iterator, List from typing.io import IO import rapidjson @@ -81,7 +81,7 @@ def json_load(datafile: IO) -> Any: """ load data with rapidjson Use this to have a consistent experience, - sete number_mode to "NM_NATIVE" for greatest speed + set number_mode to "NM_NATIVE" for greatest speed """ return rapidjson.load(datafile, number_mode=rapidjson.NM_NATIVE) @@ -202,3 +202,14 @@ def render_template_with_fallback(templatefile: str, templatefallbackfile: str, return render_template(templatefile, arguments) except TemplateNotFound: return render_template(templatefallbackfile, arguments) + + +def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]: + """ + Split lst into chunks of the size n. + :param lst: list to split into chunks + :param n: number of max elements per chunk + :return: None + """ + for chunk in range(0, len(lst), n): + yield (lst[chunk:chunk + n]) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 3186313e1..922f89c22 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -15,19 +15,21 @@ from freqtrade.configuration import TimeRange, remove_credentials, validate_conf from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data import history from freqtrade.data.btanalysis import trade_list_to_dataframe -from freqtrade.data.converter import trim_dataframe +from freqtrade.data.converter import trim_dataframes from freqtrade.data.dataprovider import DataProvider -from freqtrade.exceptions import OperationalException +from freqtrade.enums import SellType +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.mixins import LoggingMixin from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_stats) -from freqtrade.persistence import PairLocks, Trade +from freqtrade.persistence import LocalTrade, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType +from freqtrade.strategy.interface import IStrategy, SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper +from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) @@ -62,9 +64,7 @@ class Backtesting: self.all_results: Dict[str, Dict] = {} self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) - - dataprovider = DataProvider(self.config, self.exchange) - IStrategy.dp = dataprovider + self.dataprovider = DataProvider(self.config, None) if self.config.get('strategy_list', None): for strat in list(self.config['strategy_list']): @@ -95,7 +95,7 @@ class Backtesting: "PrecisionFilter not allowed for backtesting multiple strategies." ) - dataprovider.add_pairlisthandler(self.pairlists) + self.dataprovider.add_pairlisthandler(self.pairlists) self.pairlists.refresh_pairlist() if len(self.pairlists.whitelist) == 0: @@ -111,28 +111,33 @@ class Backtesting: PairLocks.timeframe = self.config['timeframe'] PairLocks.use_db = False PairLocks.reset_locks() - if self.config.get('enable_protections', False): - self.protections = ProtectionManager(self.config) + + self.wallets = Wallets(self.config, self.exchange, log=False) # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) - # Load one (first) strategy - self._set_strategy(self.strategylist[0]) def __del__(self): LoggingMixin.show_output = True PairLocks.use_db = True Trade.use_db = True - def _set_strategy(self, strategy): + def _set_strategy(self, strategy: IStrategy): """ Load strategy into backtesting """ self.strategy: IStrategy = strategy + strategy.dp = self.dataprovider # Set stoploss_on_exchange to false for backtesting, # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False + if self.config.get('enable_protections', False): + conf = self.config + if hasattr(strategy, 'protections'): + conf = deepcopy(conf) + conf['protections'] = strategy.protections + self.protections = ProtectionManager(conf) def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: """ @@ -156,7 +161,7 @@ class Backtesting: logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(max_date - min_date).days} days)..') + f'({(max_date - min_date).days} days).') # Adjust startts forward if not enough data is available timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe), @@ -171,10 +176,10 @@ class Backtesting: PairLocks.use_db = False PairLocks.timeframe = self.config['timeframe'] Trade.use_db = False - if enable_protections: - # Reset persisted data - used for protections only - PairLocks.reset_locks() - Trade.reset_trades() + PairLocks.reset_locks() + Trade.reset_trades() + self.rejected_trades = 0 + self.dataprovider.clear_cache() def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]: """ @@ -188,8 +193,9 @@ class Backtesting: data: Dict = {} # Create dict with data for pair, pair_data in processed.items(): - pair_data.loc[:, 'buy'] = 0 # cleanup from previous run - pair_data.loc[:, 'sell'] = 0 # cleanup from previous run + if not pair_data.empty: + pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist + pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist df_analyzed = self.strategy.advise_sell( self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() @@ -203,16 +209,22 @@ class Backtesting: # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) - data[pair] = [x for x in df_analyzed.itertuples(index=False, name=None)] + data[pair] = df_analyzed.values.tolist() return data - def _get_close_rate(self, sell_row: Tuple, trade: Trade, sell: SellCheckTuple, + def _get_close_rate(self, sell_row: Tuple, trade: LocalTrade, sell: SellCheckTuple, trade_dur: int) -> float: """ Get close rate for backtesting result """ # Special handling if high or low hit STOP_LOSS or ROI if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): + if trade.stop_loss > sell_row[HIGH_IDX]: + # our stoploss was already higher than candle high, + # possibly due to a cancelled trade exit. + # sell at open price. + return sell_row[OPEN_IDX] + # Set close_rate to stoploss return trade.stop_loss elif sell.sell_type == (SellType.ROI): @@ -238,7 +250,7 @@ class Backtesting: # Use the maximum between close_rate and low as we # cannot sell outside of a candle. # Applies when a new ROI setting comes in place and the whole candle is above that. - return max(close_rate, sell_row[LOW_IDX]) + return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX]) else: # This should not be reached... @@ -246,24 +258,67 @@ class Backtesting: else: return sell_row[OPEN_IDX] - def _get_sell_trade_entry(self, trade: Trade, sell_row: Tuple) -> Optional[Trade]: + def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]: - sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], sell_row[DATE_IDX], - sell_row[BUY_IDX], sell_row[SELL_IDX], + sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore + sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX], + sell_row[SELL_IDX], low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX]) + if sell.sell_flag: - trade_dur = int((sell_row[DATE_IDX] - trade.open_date).total_seconds() // 60) + trade.close_date = sell_row[DATE_IDX].to_pydatetime() + trade.sell_reason = sell.sell_reason + trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60) closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) - trade.close_date = sell_row[DATE_IDX] - trade.sell_reason = sell.sell_type + # Confirm trade exit: + time_in_force = self.strategy.order_time_in_force['sell'] + if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( + pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount, + rate=closerate, + time_in_force=time_in_force, + sell_reason=sell.sell_reason, + current_time=sell_row[DATE_IDX].to_pydatetime()): + return None + trade.close(closerate, show_msg=False) return trade return None - def handle_left_open(self, open_trades: Dict[str, List[Trade]], - data: Dict[str, List[Tuple]]) -> List[Trade]: + def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]: + try: + stake_amount = self.wallets.get_trade_stake_amount(pair, None) + except DependencyException: + return None + min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05) + + order_type = self.strategy.order_types['buy'] + time_in_force = self.strategy.order_time_in_force['sell'] + # Confirm trade entry: + if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( + pair=pair, order_type=order_type, amount=stake_amount, rate=row[OPEN_IDX], + time_in_force=time_in_force, current_time=row[DATE_IDX].to_pydatetime()): + return None + + if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount): + # Enter trade + trade = LocalTrade( + pair=pair, + open_rate=row[OPEN_IDX], + open_date=row[DATE_IDX].to_pydatetime(), + stake_amount=stake_amount, + amount=round(stake_amount / row[OPEN_IDX], 8), + fee_open=self.fee, + fee_close=self.fee, + is_open=True, + exchange='backtesting', + ) + return trade + return None + + def handle_left_open(self, open_trades: Dict[str, List[LocalTrade]], + data: Dict[str, List[Tuple]]) -> List[LocalTrade]: """ Handling of left open trades at the end of backtesting """ @@ -273,17 +328,28 @@ class Backtesting: for trade in open_trades[pair]: sell_row = data[pair][-1] - trade.close_date = sell_row[DATE_IDX] - trade.sell_reason = SellType.FORCE_SELL + trade.close_date = sell_row[DATE_IDX].to_pydatetime() + trade.sell_reason = SellType.FORCE_SELL.value trade.close(sell_row[OPEN_IDX], show_msg=False) - trade.is_open = True - trades.append(trade) + LocalTrade.close_bt_trade(trade) + # Deepcopy object to have wallets update correctly + trade1 = deepcopy(trade) + trade1.is_open = True + trades.append(trade1) return trades - def backtest(self, processed: Dict, stake_amount: float, + def trade_slot_available(self, max_open_trades: int, open_trade_count: int) -> bool: + # Always allow trades when max_open_trades is enabled. + if max_open_trades <= 0 or open_trade_count < max_open_trades: + return True + # Rejected trade + self.rejected_trades += 1 + return False + + def backtest(self, processed: Dict, start_date: datetime, end_date: datetime, max_open_trades: int = 0, position_stacking: bool = False, - enable_protections: bool = False) -> DataFrame: + enable_protections: bool = False) -> Dict[str, Any]: """ Implement backtesting functionality @@ -292,7 +358,6 @@ class Backtesting: Avoid extensive logging in this method and functions it calls. :param processed: a processed dictionary with format {pair, data} - :param stake_amount: amount to use for each trade :param start_date: backtesting timerange start datetime :param end_date: backtesting timerange end datetime :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited @@ -300,22 +365,22 @@ class Backtesting: :param enable_protections: Should protections be enabled? :return: DataFrame with trades (results of backtesting) """ - logger.debug(f"Run backtest, stake_amount: {stake_amount}, " - f"start_date: {start_date}, end_date: {end_date}, " - f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}" - ) - trades: List[Trade] = [] + trades: List[LocalTrade] = [] self.prepare_backtest(enable_protections) + # Update dataprovider cache + for pair, dataframe in processed.items(): + self.dataprovider._set_cached_df(pair, self.timeframe, dataframe) + # Use dict of lists with data for performance # (looping lists is a lot faster than pandas DataFrames) data: Dict = self._get_ohlcv_as_lists(processed) # Indexes per pair, so some pairs are allowed to have a missing start. - indexes: Dict = {} + indexes: Dict = defaultdict(int) tmp = start_date + timedelta(minutes=self.timeframe_min) - open_trades: Dict[str, List] = defaultdict(list) + open_trades: Dict[str, List[LocalTrade]] = defaultdict(list) open_trade_count = 0 # Loop timerange and get candle for each pair at that point in time @@ -323,11 +388,9 @@ class Backtesting: open_trade_count_start = open_trade_count for i, pair in enumerate(data): - if pair not in indexes: - indexes[pair] = 0 - + row_index = indexes[pair] try: - row = data[pair][indexes[pair]] + row = data[pair][row_index] except IndexError: # missing Data for one pair at the end. # Warnings for this are shown during data loading @@ -336,38 +399,34 @@ class Backtesting: # Waits until the time-counter reaches the start of the data for this pair. if row[DATE_IDX] > tmp: continue - indexes[pair] += 1 + + row_index += 1 + self.dataprovider._set_dataframe_max_index(row_index) + indexes[pair] = row_index # without positionstacking, we can only have one open trade per pair. # max_open_trades must be respected # don't open on the last row - if ((position_stacking or len(open_trades[pair]) == 0) - and (max_open_trades <= 0 or open_trade_count_start < max_open_trades) - and tmp != end_date - and row[BUY_IDX] == 1 and row[SELL_IDX] != 1 - and not PairLocks.is_pair_locked(pair, row[DATE_IDX])): - # Enter trade - trade = Trade( - pair=pair, - open_rate=row[OPEN_IDX], - open_date=row[DATE_IDX], - stake_amount=stake_amount, - amount=round(stake_amount / row[OPEN_IDX], 8), - fee_open=self.fee, - fee_close=self.fee, - is_open=True, - ) - # TODO: hacky workaround to avoid opening > max_open_trades - # This emulates previous behaviour - not sure if this is correct - # Prevents buying if the trade-slot was freed in this candle - open_trade_count_start += 1 - open_trade_count += 1 - # logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.") - open_trades[pair].append(trade) - Trade.trades.append(trade) + if ( + (position_stacking or len(open_trades[pair]) == 0) + and self.trade_slot_available(max_open_trades, open_trade_count_start) + and tmp != end_date + and row[BUY_IDX] == 1 + and row[SELL_IDX] != 1 + and not PairLocks.is_pair_locked(pair, row[DATE_IDX]) + ): + trade = self._enter_trade(pair, row) + if trade: + # TODO: hacky workaround to avoid opening > max_open_trades + # This emulates previous behaviour - not sure if this is correct + # Prevents buying if the trade-slot was freed in this candle + open_trade_count_start += 1 + open_trade_count += 1 + # logger.debug(f"{pair} - Emulate creation of new trade: {trade}.") + open_trades[pair].append(trade) + LocalTrade.add_bt_trade(trade) for trade in open_trades[pair]: - # since indexes has been incremented before, we need to go one step back to # also check the buying candle for sell conditions. trade_entry = self._get_sell_trade_entry(trade, row) # Sell occured @@ -375,6 +434,8 @@ class Backtesting: # logger.debug(f"{pair} - Backtesting sell {trade}") open_trade_count -= 1 open_trades[pair].remove(trade) + + LocalTrade.close_bt_trade(trade) trades.append(trade_entry) if enable_protections: self.protections.stop_per_pair(pair, row[DATE_IDX]) @@ -384,8 +445,16 @@ class Backtesting: tmp += timedelta(minutes=self.timeframe_min) trades += self.handle_left_open(open_trades, data=data) + self.wallets.update() - return trade_list_to_dataframe(trades) + results = trade_list_to_dataframe(trades) + return { + 'results': results, + 'config': self.strategy.config, + 'locks': PairLocks.get_all_locks(), + 'rejected_signals': self.rejected_trades, + 'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']), + } def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange): logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) @@ -407,31 +476,32 @@ class Backtesting: preprocessed = self.strategy.ohlcvdata_to_dataframe(data) # Trim startup period from analyzed dataframe - for pair, df in preprocessed.items(): - preprocessed[pair] = trim_dataframe(df, timerange) - min_date, max_date = history.get_timerange(preprocessed) + preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup) + if not preprocessed: + raise OperationalException( + "No data left after adjusting for startup candles.") + + min_date, max_date = history.get_timerange(preprocessed) logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(max_date - min_date).days} days)..') + f'({(max_date - min_date).days} days).') # Execute backtest and store results results = self.backtest( processed=preprocessed, - stake_amount=self.config['stake_amount'], - start_date=min_date.datetime, - end_date=max_date.datetime, + start_date=min_date, + end_date=max_date, max_open_trades=max_open_trades, position_stacking=self.config.get('position_stacking', False), enable_protections=self.config.get('enable_protections', False), ) backtest_end_time = datetime.now(timezone.utc) - self.all_results[self.strategy.get_strategy_name()] = { - 'results': results, - 'config': self.strategy.config, - 'locks': PairLocks.locks, + results.update({ 'backtest_start_time': int(backtest_start_time.timestamp()), 'backtest_end_time': int(backtest_end_time.timestamp()), - } + }) + self.all_results[self.strategy.get_strategy_name()] = results + return min_date, max_date def start(self) -> None: @@ -442,17 +512,16 @@ class Backtesting: data: Dict[str, Any] = {} data, timerange = self.load_bt_data() + logger.info("Dataload complete. Calculating indicators") - min_date = None - max_date = None for strat in self.strategylist: min_date, max_date = self.backtest_one_strategy(strat, data, timerange) + if len(self.strategylist) > 0: + stats = generate_backtest_stats(data, self.all_results, + min_date=min_date, max_date=max_date) - stats = generate_backtest_stats(data, self.all_results, - min_date=min_date, max_date=max_date) + if self.config.get('export', False): + store_backtest_stats(self.config['exportfilename'], stats) - if self.config.get('export', False): - store_backtest_stats(self.config['exportfilename'], stats) - - # Show backtest results - show_backtest_results(self.config, stats) + # Show backtest results + show_backtest_results(self.config, stats) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index a5f505bee..aab7def05 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -44,7 +44,7 @@ class EdgeCli: 'timerange') is None else str(self.config.get('timerange'))) def start(self) -> None: - result = self.edge.calculate() + result = self.edge.calculate(self.config['exchange']['pair_whitelist']) if result: print('') # blank line for readability print(generate_edge_table(self.edge._cached_pairs)) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d0cdceaeb..c2b2b93cb 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,38 +4,34 @@ This module contains the hyperopt logic """ -import io -import locale import logging import random import warnings -from collections import OrderedDict -from datetime import datetime +from datetime import datetime, timezone from math import ceil -from operator import itemgetter from pathlib import Path -from pprint import pformat from typing import Any, Dict, List, Optional +import numpy as np import progressbar import rapidjson -import tabulate from colorama import Fore, Style from colorama import init as colorama_init from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects -from pandas import DataFrame, isna, json_normalize +from pandas import DataFrame from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN -from freqtrade.data.converter import trim_dataframe +from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange -from freqtrade.exceptions import OperationalException -from freqtrade.misc import file_dump_json, plural, round_dict +from freqtrade.misc import file_dump_json, plural from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules +from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 +from freqtrade.optimize.hyperopt_tools import HyperoptTools +from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver -from freqtrade.strategy import IStrategy # Suppress scikit-learn FutureWarnings from skopt @@ -66,19 +62,33 @@ class Hyperopt: hyperopt = Hyperopt(config) hyperopt.start() """ + custom_hyperopt: IHyperOpt def __init__(self, config: Dict[str, Any]) -> None: + self.buy_space: List[Dimension] = [] + self.sell_space: List[Dimension] = [] + self.roi_space: List[Dimension] = [] + self.stoploss_space: List[Dimension] = [] + self.trailing_space: List[Dimension] = [] + self.dimensions: List[Dimension] = [] + self.config = config self.backtesting = Backtesting(self.config) - self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) + if not self.config.get('hyperopt'): + self.custom_hyperopt = HyperOptAuto(self.config) + else: + self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) + self.backtesting._set_strategy(self.backtesting.strategylist[0]) + self.custom_hyperopt.strategy = self.backtesting.strategy self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config) self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") - self.results_file = (self.config['user_data_dir'] / - 'hyperopt_results' / f'hyperopt_results_{time_now}.pickle') + strategy = str(self.config['strategy']) + self.results_file: Path = (self.config['user_data_dir'] / 'hyperopt_results' / + f'strategy_{strategy}_{time_now}.fthypt') self.data_pickle_file = (self.config['user_data_dir'] / 'hyperopt_results' / 'hyperopt_tickerdata.pkl') self.total_epochs = config.get('epochs', 0) @@ -88,9 +98,7 @@ class Hyperopt: self.clean_hyperopt() self.num_epochs_saved = 0 - - # Previous evaluations - self.epochs: List = [] + self.current_best_epoch: Optional[Dict[str, Any]] = None # Populate functions here (hasattr is slow so should not be run during "regular" operations) if hasattr(self.custom_hyperopt, 'populate_indicators'): @@ -111,7 +119,7 @@ class Hyperopt: self.max_open_trades = 0 self.position_stacking = self.config.get('position_stacking', False) - if self.has_space('sell'): + if HyperoptTools.has_space(self.config, 'sell'): # Make sure use_sell_signal is enabled if 'ask_strategy' not in self.config: self.config['ask_strategy'] = {} @@ -137,9 +145,7 @@ class Hyperopt: logger.info(f"Removing `{p}`.") p.unlink() - def _get_params_dict(self, raw_params: List[Any]) -> Dict: - - dimensions: List[Dimension] = self.dimensions + def _get_params_dict(self, dimensions: List[Dimension], raw_params: List[Any]) -> Dict: # Ensure the number of dimensions match # the number of parameters in the list. @@ -150,30 +156,30 @@ class Hyperopt: # and the values are taken from the list of parameters. return {d.name: v for d, v in zip(dimensions, raw_params)} - def _save_results(self) -> None: + def _save_result(self, epoch: Dict) -> None: """ Save hyperopt results to file + Store one line per epoch. + While not a valid json object - this allows appending easily. + :param epoch: result dictionary for this epoch. """ - num_epochs = len(self.epochs) - if num_epochs > self.num_epochs_saved: - logger.debug(f"Saving {num_epochs} {plural(num_epochs, 'epoch')}.") - dump(self.epochs, self.results_file) - self.num_epochs_saved = num_epochs - logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " - f"saved to '{self.results_file}'.") - # Store hyperopt filename - latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN) - file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)}, - log=False) + def default_parser(x): + if isinstance(x, np.integer): + return int(x) + return str(x) - @staticmethod - def _read_results(results_file: Path) -> List: - """ - Read hyperopt results from file - """ - logger.info("Reading epochs from '%s'", results_file) - data = load(results_file) - return data + with self.results_file.open('a') as f: + rapidjson.dump(epoch, f, default=default_parser, + number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN) + f.write("\n") + + self.num_epochs_saved += 1 + logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " + f"saved to '{self.results_file}'.") + # Store hyperopt filename + latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN) + file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)}, + log=False) def _get_params_details(self, params: Dict) -> Dict: """ @@ -181,118 +187,30 @@ class Hyperopt: """ result: Dict = {} - if self.has_space('buy'): - result['buy'] = {p.name: params.get(p.name) - for p in self.hyperopt_space('buy')} - if self.has_space('sell'): - result['sell'] = {p.name: params.get(p.name) - for p in self.hyperopt_space('sell')} - if self.has_space('roi'): - result['roi'] = self.custom_hyperopt.generate_roi_table(params) - if self.has_space('stoploss'): - result['stoploss'] = {p.name: params.get(p.name) - for p in self.hyperopt_space('stoploss')} - if self.has_space('trailing'): + if HyperoptTools.has_space(self.config, 'buy'): + result['buy'] = {p.name: params.get(p.name) for p in self.buy_space} + if HyperoptTools.has_space(self.config, 'sell'): + result['sell'] = {p.name: params.get(p.name) for p in self.sell_space} + if HyperoptTools.has_space(self.config, 'roi'): + result['roi'] = {str(k): v for k, v in + self.custom_hyperopt.generate_roi_table(params).items()} + if HyperoptTools.has_space(self.config, 'stoploss'): + result['stoploss'] = {p.name: params.get(p.name) for p in self.stoploss_space} + if HyperoptTools.has_space(self.config, 'trailing'): result['trailing'] = self.custom_hyperopt.generate_trailing_params(params) return result - @staticmethod - def print_epoch_details(results, total_epochs: int, print_json: bool, - no_header: bool = False, header_str: str = None) -> None: - """ - Display details of the hyperopt result - """ - params = results.get('params_details', {}) - - # Default header string - if header_str is None: - header_str = "Best result" - - if not no_header: - explanation_str = Hyperopt._format_explanation_string(results, total_epochs) - print(f"\n{header_str}:\n\n{explanation_str}\n") - - if print_json: - result_dict: Dict = {} - for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: - Hyperopt._params_update_for_json(result_dict, params, s) - print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE)) - - else: - Hyperopt._params_pretty_print(params, 'buy', "Buy hyperspace params:") - Hyperopt._params_pretty_print(params, 'sell', "Sell hyperspace params:") - Hyperopt._params_pretty_print(params, 'roi', "ROI table:") - Hyperopt._params_pretty_print(params, 'stoploss', "Stoploss:") - Hyperopt._params_pretty_print(params, 'trailing', "Trailing stop:") - - @staticmethod - def _params_update_for_json(result_dict, params, space: str) -> None: - if space in params: - space_params = Hyperopt._space_params(params, space) - if space in ['buy', 'sell']: - result_dict.setdefault('params', {}).update(space_params) - elif space == 'roi': - # TODO: get rid of OrderedDict when support for python 3.6 will be - # dropped (dicts keep the order as the language feature) - - # Convert keys in min_roi dict to strings because - # rapidjson cannot dump dicts with integer keys... - # OrderedDict is used to keep the numeric order of the items - # in the dict. - result_dict['minimal_roi'] = OrderedDict( - (str(k), v) for k, v in space_params.items() - ) - else: # 'stoploss', 'trailing' - result_dict.update(space_params) - - @staticmethod - def _params_pretty_print(params, space: str, header: str) -> None: - if space in params: - space_params = Hyperopt._space_params(params, space, 5) - params_result = f"\n# {header}\n" - if space == 'stoploss': - params_result += f"stoploss = {space_params.get('stoploss')}" - elif space == 'roi': - # TODO: get rid of OrderedDict when support for python 3.6 will be - # dropped (dicts keep the order as the language feature) - minimal_roi_result = rapidjson.dumps( - OrderedDict( - (str(k), v) for k, v in space_params.items() - ), - default=str, indent=4, number_mode=rapidjson.NM_NATIVE) - params_result += f"minimal_roi = {minimal_roi_result}" - elif space == 'trailing': - - for k, v in space_params.items(): - params_result += f'{k} = {v}\n' - - else: - params_result += f"{space}_params = {pformat(space_params, indent=4)}" - params_result = params_result.replace("}", "\n}").replace("{", "{\n ") - - params_result = params_result.replace("\n", "\n ") - print(params_result) - - @staticmethod - def _space_params(params, space: str, r: int = None) -> Dict: - d = params[space] - # Round floats to `r` digits after the decimal point if requested - return round_dict(d, r) if r else d - - @staticmethod - def is_best_loss(results, current_best_loss: float) -> bool: - return results['loss'] < current_best_loss - def print_results(self, results) -> None: """ Log results if it is better than any previous evaluation + TODO: this should be moved to HyperoptTools too """ is_best = results['is_best'] if self.print_all or is_best: print( - self.get_result_table( + HyperoptTools.get_result_table( self.config, results, self.total_epochs, self.print_all, self.print_colorized, self.hyperopt_table_header @@ -300,229 +218,58 @@ class Hyperopt: ) self.hyperopt_table_header = 2 - @staticmethod - def _format_explanation_string(results, total_epochs) -> str: - return (("*" if results['is_initial_point'] else " ") + - f"{results['current_epoch']:5d}/{total_epochs}: " + - f"{results['results_explanation']} " + - f"Objective: {results['loss']:.5f}") - - @staticmethod - def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, - print_colorized: bool, remove_header: int) -> str: + def init_spaces(self): """ - Log result table + Assign the dimensions in the hyperoptimization space. """ - if not results: - return '' - tabulate.PRESERVE_WHITESPACE = True - - trials = json_normalize(results, max_level=1) - trials['Best'] = '' - if 'results_metrics.winsdrawslosses' not in trials.columns: - # Ensure compatibility with older versions of hyperopt results - trials['results_metrics.winsdrawslosses'] = 'N/A' - - trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', - 'results_metrics.winsdrawslosses', - 'results_metrics.avg_profit', 'results_metrics.total_profit', - 'results_metrics.profit', 'results_metrics.duration', - 'loss', 'is_initial_point', 'is_best']] - trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', - 'Total profit', 'Profit', 'Avg duration', 'Objective', - 'is_initial_point', 'is_best'] - trials['is_profit'] = False - trials.loc[trials['is_initial_point'], 'Best'] = '* ' - trials.loc[trials['is_best'], 'Best'] = 'Best' - trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best' - trials.loc[trials['Total profit'] > 0, 'is_profit'] = True - trials['Trades'] = trials['Trades'].astype(str) - - trials['Epoch'] = trials['Epoch'].apply( - lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs) - ) - trials['Avg profit'] = trials['Avg profit'].apply( - lambda x: '{:,.2f}%'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ') - ) - trials['Avg duration'] = trials['Avg duration'].apply( - lambda x: '{:,.1f} m'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ') - ) - trials['Objective'] = trials['Objective'].apply( - lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ') - ) - - trials['Profit'] = trials.apply( - lambda x: '{:,.8f} {} {}'.format( - x['Total profit'], config['stake_currency'], - '({:,.2f}%)'.format(x['Profit']).rjust(10, ' ') - ).rjust(25+len(config['stake_currency'])) - if x['Total profit'] != 0.0 else '--'.rjust(25+len(config['stake_currency'])), - axis=1 - ) - trials = trials.drop(columns=['Total profit']) - - if print_colorized: - for i in range(len(trials)): - if trials.loc[i]['is_profit']: - for j in range(len(trials.loc[i])-3): - trials.iat[i, j] = "{}{}{}".format(Fore.GREEN, - str(trials.loc[i][j]), Fore.RESET) - if trials.loc[i]['is_best'] and highlight_best: - for j in range(len(trials.loc[i])-3): - trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT, - str(trials.loc[i][j]), Style.RESET_ALL) - - trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) - if remove_header > 0: - table = tabulate.tabulate( - trials.to_dict(orient='list'), tablefmt='orgtbl', - headers='keys', stralign="right" - ) - - table = table.split("\n", remove_header)[remove_header] - elif remove_header < 0: - table = tabulate.tabulate( - trials.to_dict(orient='list'), tablefmt='psql', - headers='keys', stralign="right" - ) - table = "\n".join(table.split("\n")[0:remove_header]) - else: - table = tabulate.tabulate( - trials.to_dict(orient='list'), tablefmt='psql', - headers='keys', stralign="right" - ) - return table - - @staticmethod - def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool, - csv_file: str) -> None: - """ - Log result to csv-file - """ - if not results: - return - - # Verification for overwrite - if Path(csv_file).is_file(): - logger.error(f"CSV file already exists: {csv_file}") - return - - try: - io.open(csv_file, 'w+').close() - except IOError: - logger.error(f"Failed to create CSV file: {csv_file}") - return - - trials = json_normalize(results, max_level=1) - trials['Best'] = '' - trials['Stake currency'] = config['stake_currency'] - - base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count', - 'results_metrics.avg_profit', 'results_metrics.total_profit', - 'Stake currency', 'results_metrics.profit', 'results_metrics.duration', - 'loss', 'is_initial_point', 'is_best'] - param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()] - trials = trials[base_metrics + param_metrics] - - base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency', - 'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best'] - param_columns = list(results[0]['params_dict'].keys()) - trials.columns = base_columns + param_columns - - trials['is_profit'] = False - trials.loc[trials['is_initial_point'], 'Best'] = '*' - trials.loc[trials['is_best'], 'Best'] = 'Best' - trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best' - trials.loc[trials['Total profit'] > 0, 'is_profit'] = True - trials['Epoch'] = trials['Epoch'].astype(str) - trials['Trades'] = trials['Trades'].astype(str) - - trials['Total profit'] = trials['Total profit'].apply( - lambda x: '{:,.8f}'.format(x) if x != 0.0 else "" - ) - trials['Profit'] = trials['Profit'].apply( - lambda x: '{:,.2f}'.format(x) if not isna(x) else "" - ) - trials['Avg profit'] = trials['Avg profit'].apply( - lambda x: '{:,.2f}%'.format(x) if not isna(x) else "" - ) - trials['Avg duration'] = trials['Avg duration'].apply( - lambda x: '{:,.1f} m'.format(x) if not isna(x) else "" - ) - trials['Objective'] = trials['Objective'].apply( - lambda x: '{:,.5f}'.format(x) if x != 100000 else "" - ) - - trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) - trials.to_csv(csv_file, index=False, header=True, mode='w', encoding='UTF-8') - logger.info(f"CSV file created: {csv_file}") - - def has_space(self, space: str) -> bool: - """ - Tell if the space value is contained in the configuration - """ - # The 'trailing' space is not included in the 'default' set of spaces - if space == 'trailing': - return any(s in self.config['spaces'] for s in [space, 'all']) - else: - return any(s in self.config['spaces'] for s in [space, 'all', 'default']) - - def hyperopt_space(self, space: Optional[str] = None) -> List[Dimension]: - """ - Return the dimensions in the hyperoptimization space. - :param space: Defines hyperspace to return dimensions for. - If None, then the self.has_space() will be used to return dimensions - for all hyperspaces used. - """ - spaces: List[Dimension] = [] - - if space == 'buy' or (space is None and self.has_space('buy')): + if HyperoptTools.has_space(self.config, 'buy'): logger.debug("Hyperopt has 'buy' space") - spaces += self.custom_hyperopt.indicator_space() + self.buy_space = self.custom_hyperopt.indicator_space() - if space == 'sell' or (space is None and self.has_space('sell')): + if HyperoptTools.has_space(self.config, 'sell'): logger.debug("Hyperopt has 'sell' space") - spaces += self.custom_hyperopt.sell_indicator_space() + self.sell_space = self.custom_hyperopt.sell_indicator_space() - if space == 'roi' or (space is None and self.has_space('roi')): + if HyperoptTools.has_space(self.config, 'roi'): logger.debug("Hyperopt has 'roi' space") - spaces += self.custom_hyperopt.roi_space() + self.roi_space = self.custom_hyperopt.roi_space() - if space == 'stoploss' or (space is None and self.has_space('stoploss')): + if HyperoptTools.has_space(self.config, 'stoploss'): logger.debug("Hyperopt has 'stoploss' space") - spaces += self.custom_hyperopt.stoploss_space() + self.stoploss_space = self.custom_hyperopt.stoploss_space() - if space == 'trailing' or (space is None and self.has_space('trailing')): + if HyperoptTools.has_space(self.config, 'trailing'): logger.debug("Hyperopt has 'trailing' space") - spaces += self.custom_hyperopt.trailing_space() - - return spaces + self.trailing_space = self.custom_hyperopt.trailing_space() + self.dimensions = (self.buy_space + self.sell_space + self.roi_space + + self.stoploss_space + self.trailing_space) def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict: """ Used Optimize function. Called once per epoch to optimize whatever is configured. Keep this function as optimized as possible! """ - params_dict = self._get_params_dict(raw_params) - params_details = self._get_params_details(params_dict) + backtest_start_time = datetime.now(timezone.utc) + params_dict = self._get_params_dict(self.dimensions, raw_params) - if self.has_space('roi'): + # Apply parameters + if HyperoptTools.has_space(self.config, 'roi'): self.backtesting.strategy.minimal_roi = ( # type: ignore self.custom_hyperopt.generate_roi_table(params_dict)) - if self.has_space('buy'): + if HyperoptTools.has_space(self.config, 'buy'): self.backtesting.strategy.advise_buy = ( # type: ignore self.custom_hyperopt.buy_strategy_generator(params_dict)) - if self.has_space('sell'): + if HyperoptTools.has_space(self.config, 'sell'): self.backtesting.strategy.advise_sell = ( # type: ignore self.custom_hyperopt.sell_strategy_generator(params_dict)) - if self.has_space('stoploss'): + if HyperoptTools.has_space(self.config, 'stoploss'): self.backtesting.strategy.stoploss = params_dict['stoploss'] - if self.has_space('trailing'): + if HyperoptTools.has_space(self.config, 'trailing'): d = self.custom_hyperopt.generate_trailing_params(params_dict) self.backtesting.strategy.trailing_stop = d['trailing_stop'] self.backtesting.strategy.trailing_stop_positive = d['trailing_stop_positive'] @@ -531,30 +278,42 @@ class Hyperopt: self.backtesting.strategy.trailing_only_offset_is_reached = \ d['trailing_only_offset_is_reached'] - processed = load(self.data_pickle_file) - - min_date, max_date = get_timerange(processed) - - backtesting_results = self.backtesting.backtest( + with self.data_pickle_file.open('rb') as f: + processed = load(f, mmap_mode='r') + bt_results = self.backtesting.backtest( processed=processed, - stake_amount=self.config['stake_amount'], - start_date=min_date.datetime, - end_date=max_date.datetime, + start_date=self.min_date, + end_date=self.max_date, max_open_trades=self.max_open_trades, position_stacking=self.position_stacking, enable_protections=self.config.get('enable_protections', False), - ) - return self._get_results_dict(backtesting_results, min_date, max_date, - params_dict, params_details) + backtest_end_time = datetime.now(timezone.utc) + bt_results.update({ + 'backtest_start_time': int(backtest_start_time.timestamp()), + 'backtest_end_time': int(backtest_end_time.timestamp()), + }) + + return self._get_results_dict(bt_results, self.min_date, self.max_date, + params_dict, + processed=processed) def _get_results_dict(self, backtesting_results, min_date, max_date, - params_dict, params_details): - results_metrics = self._calculate_results_metrics(backtesting_results) - results_explanation = self._format_results_explanation_string(results_metrics) + params_dict, processed: Dict[str, DataFrame] + ) -> Dict[str, Any]: + params_details = self._get_params_details(params_dict) - trade_count = results_metrics['trade_count'] - total_profit = results_metrics['total_profit'] + strat_stats = generate_strategy_stats( + processed, self.backtesting.strategy.get_strategy_name(), + backtesting_results, min_date, max_date, market_change=0 + ) + results_explanation = HyperoptTools.format_results_explanation_string( + strat_stats, self.config['stake_currency']) + + not_optimized = self.backtesting.strategy.get_params_dict() + + trade_count = strat_stats['total_trades'] + total_profit = strat_stats['profit_total'] # If this evaluation contains too short amount of trades to be # interesting -- consider it as 'bad' (assigned max. loss value) @@ -562,49 +321,20 @@ class Hyperopt: # path. We do not want to optimize 'hodl' strategies. loss: float = MAX_LOSS if trade_count >= self.config['hyperopt_min_trades']: - loss = self.calculate_loss(results=backtesting_results, trade_count=trade_count, - min_date=min_date.datetime, max_date=max_date.datetime) + loss = self.calculate_loss(results=backtesting_results['results'], + trade_count=trade_count, + min_date=min_date, max_date=max_date, + config=self.config, processed=processed) return { 'loss': loss, 'params_dict': params_dict, 'params_details': params_details, - 'results_metrics': results_metrics, + 'params_not_optimized': not_optimized, + 'results_metrics': strat_stats, 'results_explanation': results_explanation, 'total_profit': total_profit, } - def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict: - wins = len(backtesting_results[backtesting_results['profit_ratio'] > 0]) - draws = len(backtesting_results[backtesting_results['profit_ratio'] == 0]) - losses = len(backtesting_results[backtesting_results['profit_ratio'] < 0]) - return { - 'trade_count': len(backtesting_results.index), - 'wins': wins, - 'draws': draws, - 'losses': losses, - 'winsdrawslosses': f"{wins:>4} {draws:>4} {losses:>4}", - 'avg_profit': backtesting_results['profit_ratio'].mean() * 100.0, - 'median_profit': backtesting_results['profit_ratio'].median() * 100.0, - 'total_profit': backtesting_results['profit_abs'].sum(), - 'profit': backtesting_results['profit_ratio'].sum() * 100.0, - 'duration': backtesting_results['trade_duration'].mean(), - } - - def _format_results_explanation_string(self, results_metrics: Dict) -> str: - """ - Return the formatted results explanation in a string - """ - stake_cur = self.config['stake_currency'] - return (f"{results_metrics['trade_count']:6d} trades. " - f"{results_metrics['wins']}/{results_metrics['draws']}" - f"/{results_metrics['losses']} Wins/Draws/Losses. " - f"Avg profit {results_metrics['avg_profit']: 6.2f}%. " - f"Median profit {results_metrics['median_profit']: 6.2f}%. " - f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} " - f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). " - f"Avg duration {results_metrics['duration']:5.1f} min." - ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8') - def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: return Optimizer( dimensions, @@ -620,56 +350,47 @@ class Hyperopt: return parallel(delayed( wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) - @staticmethod - def load_previous_results(results_file: Path) -> List: - """ - Load data for epochs from the file if we have one - """ - epochs: List = [] - if results_file.is_file() and results_file.stat().st_size > 0: - epochs = Hyperopt._read_results(results_file) - # Detection of some old format, without 'is_best' field saved - if epochs[0].get('is_best') is None: - raise OperationalException( - "The file with Hyperopt results is incompatible with this version " - "of Freqtrade and cannot be loaded.") - logger.info(f"Loaded {len(epochs)} previous evaluations from disk.") - return epochs - def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) + def prepare_hyperopt_data(self) -> None: + data, timerange = self.backtesting.load_bt_data() + logger.info("Dataload complete. Calculating indicators") + + preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data) + + # Trim startup period from analyzed dataframe + processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup) + + self.min_date, self.max_date = get_timerange(processed) + + logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'({(self.max_date - self.min_date).days} days)..') + + dump(processed, self.data_pickle_file) + def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None)) logger.info(f"Using optimizer random state: {self.random_state}") self.hyperopt_table_header = -1 - data, timerange = self.backtesting.load_bt_data() + # Initialize spaces ... + self.init_spaces() - preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data) - - # Trim startup period from analyzed dataframe - for pair, df in preprocessed.items(): - preprocessed[pair] = trim_dataframe(df, timerange) - min_date, max_date = get_timerange(preprocessed) - - logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' - f'({(max_date - min_date).days} days)..') - - dump(preprocessed, self.data_pickle_file) + self.prepare_hyperopt_data() # We don't need exchange instance anymore while running hyperopt - self.backtesting.exchange = None # type: ignore + self.backtesting.exchange.close() + self.backtesting.exchange._api = None # type: ignore + self.backtesting.exchange._api_async = None # type: ignore + # self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore - self.backtesting.strategy.dp = None # type: ignore - IStrategy.dp = None # type: ignore cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") config_jobs = self.config.get('hyperopt_jobs', -1) logger.info(f'Number of parallel jobs set as: {config_jobs}') - self.dimensions: List[Dimension] = self.hyperopt_space() self.opt = self.get_optimizer(self.dimensions, config_jobs) if self.print_colorized: @@ -725,7 +446,7 @@ class Hyperopt: logger.debug(f"Optimizer epoch evaluated: {val}") - is_best = self.is_best_loss(val, self.current_best_loss) + is_best = HyperoptTools.is_best_loss(val, self.current_best_loss) # This value is assigned here and not in the optimization method # to keep proper order in the list of results. That's because # evaluations can take different time. Here they are aligned in the @@ -735,25 +456,21 @@ class Hyperopt: if is_best: self.current_best_loss = val['loss'] - self.epochs.append(val) + self.current_best_epoch = val - # Save results after each best epoch and every 100 epochs - if is_best or current % 100 == 0: - self._save_results() + self._save_result(val) pbar.update(current) except KeyboardInterrupt: print('User interrupted..') - self._save_results() logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " f"saved to '{self.results_file}'.") - if self.epochs: - sorted_epochs = sorted(self.epochs, key=itemgetter('loss')) - best_epoch = sorted_epochs[0] - self.print_epoch_details(best_epoch, self.total_epochs, self.print_json) + if self.current_best_epoch: + HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, + self.print_json) else: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py new file mode 100644 index 000000000..f86204406 --- /dev/null +++ b/freqtrade/optimize/hyperopt_auto.py @@ -0,0 +1,89 @@ +""" +HyperOptAuto class. +This module implements a convenience auto-hyperopt class, which can be used together with strategies + that implement IHyperStrategy interface. +""" +from contextlib import suppress +from typing import Any, Callable, Dict, List + +from pandas import DataFrame + + +with suppress(ImportError): + from skopt.space import Dimension + +from freqtrade.optimize.hyperopt_interface import IHyperOpt + + +class HyperOptAuto(IHyperOpt): + """ + This class delegates functionality to Strategy(IHyperStrategy) and Strategy.HyperOpt classes. + Most of the time Strategy.HyperOpt class would only implement indicator_space and + sell_indicator_space methods, but other hyperopt methods can be overridden as well. + """ + + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: + def populate_buy_trend(dataframe: DataFrame, metadata: dict): + for attr_name, attr in self.strategy.enumerate_parameters('buy'): + if attr.optimize: + # noinspection PyProtectedMember + attr.value = params[attr_name] + return self.strategy.populate_buy_trend(dataframe, metadata) + + return populate_buy_trend + + def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable: + def populate_sell_trend(dataframe: DataFrame, metadata: dict): + for attr_name, attr in self.strategy.enumerate_parameters('sell'): + if attr.optimize: + # noinspection PyProtectedMember + attr.value = params[attr_name] + return self.strategy.populate_sell_trend(dataframe, metadata) + + return populate_sell_trend + + def _get_func(self, name) -> Callable: + """ + Return a function defined in Strategy.HyperOpt class, or one defined in super() class. + :param name: function name. + :return: a requested function. + """ + hyperopt_cls = getattr(self.strategy, 'HyperOpt', None) + default_func = getattr(super(), name) + if hyperopt_cls: + return getattr(hyperopt_cls, name, default_func) + else: + return default_func + + def _generate_indicator_space(self, category): + for attr_name, attr in self.strategy.enumerate_parameters(category): + if attr.optimize: + yield attr.get_space(attr_name) + + def _get_indicator_space(self, category, fallback_method_name): + indicator_space = list(self._generate_indicator_space(category)) + if len(indicator_space) > 0: + return indicator_space + else: + return self._get_func(fallback_method_name)() + + def indicator_space(self) -> List['Dimension']: + return self._get_indicator_space('buy', 'indicator_space') + + def sell_indicator_space(self) -> List['Dimension']: + return self._get_indicator_space('sell', 'sell_indicator_space') + + def generate_roi_table(self, params: Dict) -> Dict[int, float]: + return self._get_func('generate_roi_table')(params) + + def roi_space(self) -> List['Dimension']: + return self._get_func('roi_space')() + + def stoploss_space(self) -> List['Dimension']: + return self._get_func('stoploss_space')() + + def generate_trailing_params(self, params: Dict) -> Dict: + return self._get_func('generate_trailing_params')(params) + + def trailing_space(self) -> List['Dimension']: + return self._get_func('trailing_space')() diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index b8c44ed59..889854cad 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -7,11 +7,13 @@ import math from abc import ABC from typing import Any, Callable, Dict, List -from skopt.space import Categorical, Dimension, Integer, Real +from skopt.space import Categorical, Dimension, Integer from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import round_dict +from freqtrade.optimize.space import SKDecimal +from freqtrade.strategy import IStrategy logger = logging.getLogger(__name__) @@ -30,10 +32,11 @@ class IHyperOpt(ABC): Defines the mandatory structure must follow any custom hyperopt Class attributes you can use: - ticker_interval -> int: value of the ticker interval to use for the strategy + timeframe -> int: value of the timeframe to use for the strategy """ ticker_interval: str # DEPRECATED timeframe: str + strategy: IStrategy def __init__(self, config: dict) -> None: self.config = config @@ -42,36 +45,31 @@ class IHyperOpt(ABC): IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED IHyperOpt.timeframe = str(config['timeframe']) - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: """ Create a buy strategy generator. """ raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy')) - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: + def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable: """ Create a sell strategy generator. """ raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell')) - @staticmethod - def indicator_space() -> List[Dimension]: + def indicator_space(self) -> List[Dimension]: """ Create an indicator space. """ raise OperationalException(_format_exception_message('indicator_space', 'buy')) - @staticmethod - def sell_indicator_space() -> List[Dimension]: + def sell_indicator_space(self) -> List[Dimension]: """ Create a sell indicator space. """ raise OperationalException(_format_exception_message('sell_indicator_space', 'sell')) - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: + def generate_roi_table(self, params: Dict) -> Dict[int, float]: """ Create a ROI table. @@ -86,8 +84,7 @@ class IHyperOpt(ABC): return roi_table - @staticmethod - def roi_space() -> List[Dimension]: + def roi_space(self) -> List[Dimension]: """ Create a ROI space. @@ -95,7 +92,7 @@ class IHyperOpt(ABC): This method implements adaptive roi hyperspace with varied ranges for parameters which automatically adapts to the - ticker interval used. + timeframe used. It's used by Freqtrade by default, if no custom roi_space method is defined. """ @@ -107,7 +104,7 @@ class IHyperOpt(ABC): roi_t_alpha = 1.0 roi_p_alpha = 1.0 - timeframe_min = timeframe_to_minutes(IHyperOpt.ticker_interval) + timeframe_min = timeframe_to_minutes(self.timeframe) # We define here limits for the ROI space parameters automagically adapted to the # timeframe used by the bot: @@ -117,7 +114,7 @@ class IHyperOpt(ABC): # * 'roi_p' (limits for the ROI value steps) components are scaled logarithmically. # # The scaling is designed so that it maps exactly to the legacy Freqtrade roi_space() - # method for the 5m ticker interval. + # method for the 5m timeframe. roi_t_scale = timeframe_min / 5 roi_p_scale = math.log1p(timeframe_min) / math.log1p(5) roi_limits = { @@ -143,7 +140,7 @@ class IHyperOpt(ABC): 'roi_p2': roi_limits['roi_p2_min'], 'roi_p3': roi_limits['roi_p3_min'], } - logger.info(f"Min roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}") + logger.info(f"Min roi table: {round_dict(self.generate_roi_table(p), 3)}") p = { 'roi_t1': roi_limits['roi_t1_max'], 'roi_t2': roi_limits['roi_t2_max'], @@ -152,19 +149,21 @@ class IHyperOpt(ABC): 'roi_p2': roi_limits['roi_p2_max'], 'roi_p3': roi_limits['roi_p3_max'], } - logger.info(f"Max roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}") + logger.info(f"Max roi table: {round_dict(self.generate_roi_table(p), 3)}") return [ Integer(roi_limits['roi_t1_min'], roi_limits['roi_t1_max'], name='roi_t1'), Integer(roi_limits['roi_t2_min'], roi_limits['roi_t2_max'], name='roi_t2'), Integer(roi_limits['roi_t3_min'], roi_limits['roi_t3_max'], name='roi_t3'), - Real(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], name='roi_p1'), - Real(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], name='roi_p2'), - Real(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], name='roi_p3'), + SKDecimal(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], decimals=3, + name='roi_p1'), + SKDecimal(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], decimals=3, + name='roi_p2'), + SKDecimal(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], decimals=3, + name='roi_p3'), ] - @staticmethod - def stoploss_space() -> List[Dimension]: + def stoploss_space(self) -> List[Dimension]: """ Create a stoploss space. @@ -172,11 +171,10 @@ class IHyperOpt(ABC): You may override it in your custom Hyperopt class. """ return [ - Real(-0.35, -0.02, name='stoploss'), + SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'), ] - @staticmethod - def generate_trailing_params(params: Dict) -> Dict: + def generate_trailing_params(self, params: Dict) -> Dict: """ Create dict with trailing stop parameters. """ @@ -188,8 +186,7 @@ class IHyperOpt(ABC): 'trailing_only_offset_is_reached': params['trailing_only_offset_is_reached'], } - @staticmethod - def trailing_space() -> List[Dimension]: + def trailing_space(self) -> List[Dimension]: """ Create a trailing stoploss space. @@ -204,14 +201,14 @@ class IHyperOpt(ABC): # other 'trailing' hyperspace parameters. Categorical([True], name='trailing_stop'), - Real(0.01, 0.35, name='trailing_stop_positive'), + SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'), # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive', # so this intermediate parameter is used as the value of the difference between # them. The value of the 'trailing_stop_positive_offset' is constructed in the # generate_trailing_params() method. # This is similar to the hyperspace dimensions used for constructing the ROI tables. - Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'), + SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'), Categorical([True, False], name='trailing_only_offset_is_reached'), ] diff --git a/freqtrade/optimize/hyperopt_loss_interface.py b/freqtrade/optimize/hyperopt_loss_interface.py index 48407a8a8..b5aa588b2 100644 --- a/freqtrade/optimize/hyperopt_loss_interface.py +++ b/freqtrade/optimize/hyperopt_loss_interface.py @@ -5,6 +5,7 @@ This module defines the interface for the loss-function for hyperopt from abc import ABC, abstractmethod from datetime import datetime +from typing import Dict from pandas import DataFrame @@ -19,7 +20,9 @@ class IHyperOptLoss(ABC): @staticmethod @abstractmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, - min_date: datetime, max_date: datetime, *args, **kwargs) -> float: + min_date: datetime, max_date: datetime, + config: Dict, processed: Dict[str, DataFrame], + *args, **kwargs) -> float: """ Objective function, returns smaller number for better results """ diff --git a/freqtrade/optimize/hyperopt_loss_onlyprofit.py b/freqtrade/optimize/hyperopt_loss_onlyprofit.py index 33f3f5bc6..4a3cf1b3b 100644 --- a/freqtrade/optimize/hyperopt_loss_onlyprofit.py +++ b/freqtrade/optimize/hyperopt_loss_onlyprofit.py @@ -9,23 +9,11 @@ from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss -# This is assumed to be expected avg profit * expected trade count. -# For example, for 0.35% avg per trade (or 0.0035 as ratio) and 1100 trades, -# expected max profit = 3.85 -# -# Note, this is ratio. 3.85 stated above means 385Σ%, 3.0 means 300Σ%. -# -# In this implementation it's only used in calculation of the resulting value -# of the objective function as a normalization coefficient and does not -# represent any limit for profits as in the Freqtrade legacy default loss function. -EXPECTED_MAX_PROFIT = 3.0 - - class OnlyProfitHyperOptLoss(IHyperOptLoss): """ Defines the loss function for hyperopt. - This implementation takes only profit into account. + This implementation takes only absolute profit into account, not looking at any other indicator. """ @staticmethod @@ -34,5 +22,5 @@ class OnlyProfitHyperOptLoss(IHyperOptLoss): """ Objective function, returns smaller number for better results. """ - total_profit = results['profit_ratio'].sum() - return 1 - total_profit / EXPECTED_MAX_PROFIT + total_profit = results['profit_abs'].sum() + return -1 * total_profit diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py new file mode 100644 index 000000000..fb1a0f56a --- /dev/null +++ b/freqtrade/optimize/hyperopt_tools.py @@ -0,0 +1,410 @@ + +import io +import logging +from pathlib import Path +from typing import Any, Dict, List + +import rapidjson +import tabulate +from colorama import Fore, Style +from pandas import isna, json_normalize + +from freqtrade.exceptions import OperationalException +from freqtrade.misc import round_coin_value, round_dict + + +logger = logging.getLogger(__name__) + + +class HyperoptTools(): + + @staticmethod + def has_space(config: Dict[str, Any], space: str) -> bool: + """ + Tell if the space value is contained in the configuration + """ + # The 'trailing' space is not included in the 'default' set of spaces + if space == 'trailing': + return any(s in config['spaces'] for s in [space, 'all']) + else: + return any(s in config['spaces'] for s in [space, 'all', 'default']) + + @staticmethod + def _read_results_pickle(results_file: Path) -> List: + """ + Read hyperopt results from pickle file + LEGACY method - new files are written as json and cannot be read with this method. + """ + from joblib import load + + logger.info(f"Reading pickled epochs from '{results_file}'") + data = load(results_file) + return data + + @staticmethod + def _read_results(results_file: Path) -> List: + """ + Read hyperopt results from file + """ + import rapidjson + logger.info(f"Reading epochs from '{results_file}'") + with results_file.open('r') as f: + data = [rapidjson.loads(line) for line in f] + return data + + @staticmethod + def load_previous_results(results_file: Path) -> List: + """ + Load data for epochs from the file if we have one + """ + epochs: List = [] + if results_file.is_file() and results_file.stat().st_size > 0: + if results_file.suffix == '.pickle': + epochs = HyperoptTools._read_results_pickle(results_file) + else: + epochs = HyperoptTools._read_results(results_file) + # Detection of some old format, without 'is_best' field saved + if epochs[0].get('is_best') is None: + raise OperationalException( + "The file with HyperoptTools results is incompatible with this version " + "of Freqtrade and cannot be loaded.") + logger.info(f"Loaded {len(epochs)} previous evaluations from disk.") + return epochs + + @staticmethod + def show_epoch_details(results, total_epochs: int, print_json: bool, + no_header: bool = False, header_str: str = None) -> None: + """ + Display details of the hyperopt result + """ + params = results.get('params_details', {}) + non_optimized = results.get('params_not_optimized', {}) + + # Default header string + if header_str is None: + header_str = "Best result" + + if not no_header: + explanation_str = HyperoptTools._format_explanation_string(results, total_epochs) + print(f"\n{header_str}:\n\n{explanation_str}\n") + + if print_json: + result_dict: Dict = {} + for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: + HyperoptTools._params_update_for_json(result_dict, params, s) + print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE)) + + else: + HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:", + non_optimized) + HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:", + non_optimized) + HyperoptTools._params_pretty_print(params, 'roi', "ROI table:") + HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:") + HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:") + + @staticmethod + def _params_update_for_json(result_dict, params, space: str) -> None: + if space in params: + space_params = HyperoptTools._space_params(params, space) + if space in ['buy', 'sell']: + result_dict.setdefault('params', {}).update(space_params) + elif space == 'roi': + # Convert keys in min_roi dict to strings because + # rapidjson cannot dump dicts with integer keys... + result_dict['minimal_roi'] = {str(k): v for k, v in space_params.items()} + else: # 'stoploss', 'trailing' + result_dict.update(space_params) + + @staticmethod + def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None: + if space in params or space in non_optimized: + space_params = HyperoptTools._space_params(params, space, 5) + result = f"\n# {header}\n" + if space == 'stoploss': + result += f"stoploss = {space_params.get('stoploss')}" + elif space == 'roi': + minimal_roi_result = rapidjson.dumps({ + str(k): v for k, v in space_params.items() + }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE) + result += f"minimal_roi = {minimal_roi_result}" + elif space == 'trailing': + + for k, v in space_params.items(): + result += f'{k} = {v}\n' + + else: + no_params = HyperoptTools._space_params(non_optimized, space, 5) + + result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}" + + result = result.replace("\n", "\n ") + print(result) + + @staticmethod + def _space_params(params, space: str, r: int = None) -> Dict: + d = params.get(space) + if d: + # Round floats to `r` digits after the decimal point if requested + return round_dict(d, r) if r else d + return {} + + @staticmethod + def _pprint(params, non_optimized, indent: int = 4): + """ + Pretty-print hyperopt results (based on 2 dicts - with add. comment) + """ + p = params.copy() + p.update(non_optimized) + result = '{\n' + + for k, param in p.items(): + result += " " * indent + f'"{k}": ' + result += f'"{param}",' if isinstance(param, str) else f'{param},' + if k in non_optimized: + result += " # value loaded from strategy" + result += "\n" + result += '}' + return result + + @staticmethod + def is_best_loss(results, current_best_loss: float) -> bool: + return bool(results['loss'] < current_best_loss) + + @staticmethod + def format_results_explanation_string(results_metrics: Dict, stake_currency: str) -> str: + """ + Return the formatted results explanation in a string + """ + return (f"{results_metrics['total_trades']:6d} trades. " + f"{results_metrics['wins']}/{results_metrics['draws']}" + f"/{results_metrics['losses']} Wins/Draws/Losses. " + f"Avg profit {results_metrics['profit_mean'] * 100: 6.2f}%. " + f"Median profit {results_metrics['profit_median'] * 100: 6.2f}%. " + f"Total profit {results_metrics['profit_total_abs']: 11.8f} {stake_currency} " + f"({results_metrics['profit_total'] * 100: 7.2f}%). " + f"Avg duration {results_metrics['holding_avg']} min." + ) + + @staticmethod + def _format_explanation_string(results, total_epochs) -> str: + return (("*" if results['is_initial_point'] else " ") + + f"{results['current_epoch']:5d}/{total_epochs}: " + + f"{results['results_explanation']} " + + f"Objective: {results['loss']:.5f}") + + @staticmethod + def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> str: + + trials['Best'] = '' + + if 'results_metrics.winsdrawslosses' not in trials.columns: + # Ensure compatibility with older versions of hyperopt results + trials['results_metrics.winsdrawslosses'] = 'N/A' + + if not has_drawdown: + # Ensure compatibility with older versions of hyperopt results + trials['results_metrics.max_drawdown_abs'] = None + trials['results_metrics.max_drawdown'] = None + + if not legacy_mode: + # New mode, using backtest result for metrics + trials['results_metrics.winsdrawslosses'] = trials.apply( + lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} " + f"{x['results_metrics.losses']:>4}", axis=1) + trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades', + 'results_metrics.winsdrawslosses', + 'results_metrics.profit_mean', 'results_metrics.profit_total_abs', + 'results_metrics.profit_total', 'results_metrics.holding_avg', + 'results_metrics.max_drawdown', 'results_metrics.max_drawdown_abs', + 'loss', 'is_initial_point', 'is_best']] + + else: + # Legacy mode + trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', + 'results_metrics.winsdrawslosses', 'results_metrics.avg_profit', + 'results_metrics.total_profit', 'results_metrics.profit', + 'results_metrics.duration', 'results_metrics.max_drawdown', + 'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point', + 'is_best']] + + trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', + 'Total profit', 'Profit', 'Avg duration', 'Max Drawdown', + 'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'] + + return trials + + @staticmethod + def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, + print_colorized: bool, remove_header: int) -> str: + """ + Log result table + """ + if not results: + return '' + + tabulate.PRESERVE_WHITESPACE = True + trials = json_normalize(results, max_level=1) + + legacy_mode = 'results_metrics.total_trades' not in trials + has_drawdown = 'results_metrics.max_drawdown_abs' in trials.columns + + trials = HyperoptTools.prepare_trials_columns(trials, legacy_mode, has_drawdown) + + trials['is_profit'] = False + trials.loc[trials['is_initial_point'], 'Best'] = '* ' + trials.loc[trials['is_best'], 'Best'] = 'Best' + trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best' + trials.loc[trials['Total profit'] > 0, 'is_profit'] = True + trials['Trades'] = trials['Trades'].astype(str) + perc_multi = 1 if legacy_mode else 100 + trials['Epoch'] = trials['Epoch'].apply( + lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs) + ) + trials['Avg profit'] = trials['Avg profit'].apply( + lambda x: f'{x * perc_multi:,.2f}%'.rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ') + ) + trials['Avg duration'] = trials['Avg duration'].apply( + lambda x: f'{x:,.1f} m'.rjust(7, ' ') if isinstance(x, float) else f"{x}" + if not isna(x) else "--".rjust(7, ' ') + ) + trials['Objective'] = trials['Objective'].apply( + lambda x: f'{x:,.5f}'.rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ') + ) + + stake_currency = config['stake_currency'] + + if has_drawdown: + trials['Max Drawdown'] = trials.apply( + lambda x: '{} {}'.format( + round_coin_value(x['max_drawdown_abs'], stake_currency), + '({:,.2f}%)'.format(x['Max Drawdown'] * perc_multi).rjust(10, ' ') + ).rjust(25 + len(stake_currency)) + if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)), + axis=1 + ) + else: + trials = trials.drop(columns=['Max Drawdown']) + + trials = trials.drop(columns=['max_drawdown_abs']) + + trials['Profit'] = trials.apply( + lambda x: '{} {}'.format( + round_coin_value(x['Total profit'], stake_currency), + '({:,.2f}%)'.format(x['Profit'] * perc_multi).rjust(10, ' ') + ).rjust(25+len(stake_currency)) + if x['Total profit'] != 0.0 else '--'.rjust(25+len(stake_currency)), + axis=1 + ) + trials = trials.drop(columns=['Total profit']) + + if print_colorized: + for i in range(len(trials)): + if trials.loc[i]['is_profit']: + for j in range(len(trials.loc[i])-3): + trials.iat[i, j] = "{}{}{}".format(Fore.GREEN, + str(trials.loc[i][j]), Fore.RESET) + if trials.loc[i]['is_best'] and highlight_best: + for j in range(len(trials.loc[i])-3): + trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT, + str(trials.loc[i][j]), Style.RESET_ALL) + + trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) + if remove_header > 0: + table = tabulate.tabulate( + trials.to_dict(orient='list'), tablefmt='orgtbl', + headers='keys', stralign="right" + ) + + table = table.split("\n", remove_header)[remove_header] + elif remove_header < 0: + table = tabulate.tabulate( + trials.to_dict(orient='list'), tablefmt='psql', + headers='keys', stralign="right" + ) + table = "\n".join(table.split("\n")[0:remove_header]) + else: + table = tabulate.tabulate( + trials.to_dict(orient='list'), tablefmt='psql', + headers='keys', stralign="right" + ) + return table + + @staticmethod + def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool, + csv_file: str) -> None: + """ + Log result to csv-file + """ + if not results: + return + + # Verification for overwrite + if Path(csv_file).is_file(): + logger.error(f"CSV file already exists: {csv_file}") + return + + try: + io.open(csv_file, 'w+').close() + except IOError: + logger.error(f"Failed to create CSV file: {csv_file}") + return + + trials = json_normalize(results, max_level=1) + trials['Best'] = '' + trials['Stake currency'] = config['stake_currency'] + + if 'results_metrics.total_trades' in trials: + base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades', + 'results_metrics.profit_mean', 'results_metrics.profit_median', + 'results_metrics.profit_total', + 'Stake currency', + 'results_metrics.profit_total_abs', 'results_metrics.holding_avg', + 'loss', 'is_initial_point', 'is_best'] + perc_multi = 100 + else: + perc_multi = 1 + base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count', + 'results_metrics.avg_profit', 'results_metrics.median_profit', + 'results_metrics.total_profit', + 'Stake currency', 'results_metrics.profit', 'results_metrics.duration', + 'loss', 'is_initial_point', 'is_best'] + param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()] + trials = trials[base_metrics + param_metrics] + + base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Median profit', 'Total profit', + 'Stake currency', 'Profit', 'Avg duration', 'Objective', + 'is_initial_point', 'is_best'] + param_columns = list(results[0]['params_dict'].keys()) + trials.columns = base_columns + param_columns + + trials['is_profit'] = False + trials.loc[trials['is_initial_point'], 'Best'] = '*' + trials.loc[trials['is_best'], 'Best'] = 'Best' + trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best' + trials.loc[trials['Total profit'] > 0, 'is_profit'] = True + trials['Epoch'] = trials['Epoch'].astype(str) + trials['Trades'] = trials['Trades'].astype(str) + trials['Median profit'] = trials['Median profit'] * perc_multi + + trials['Total profit'] = trials['Total profit'].apply( + lambda x: f'{x:,.8f}' if x != 0.0 else "" + ) + trials['Profit'] = trials['Profit'].apply( + lambda x: f'{x:,.2f}' if not isna(x) else "" + ) + trials['Avg profit'] = trials['Avg profit'].apply( + lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else "" + ) + if perc_multi == 1: + trials['Avg duration'] = trials['Avg duration'].apply( + lambda x: f'{x:,.1f} m' if isinstance( + x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else "" + ) + trials['Objective'] = trials['Objective'].apply( + lambda x: f'{x:,.5f}' if x != 100000 else "" + ) + + trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) + trials.to_csv(csv_file, index=False, header=True, mode='w', encoding='UTF-8') + logger.info(f"CSV file created: {csv_file}") diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 118253e86..84e052ac4 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -3,13 +3,13 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Union -from arrow import Arrow from numpy import int64 from pandas import DataFrame from tabulate import tabulate -from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN -from freqtrade.data.btanalysis import calculate_market_change, calculate_max_drawdown +from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN, UNLIMITED_STAKE_AMOUNT +from freqtrade.data.btanalysis import (calculate_csum, calculate_market_change, + calculate_max_drawdown) from freqtrade.misc import decimals_per_coin, file_dump_json, round_coin_value @@ -43,7 +43,7 @@ def _get_line_floatfmt(stake_currency: str) -> List[str]: Generate floatformat (goes in line with _generate_result_line()) """ return ['s', 'd', '.2f', '.2f', f'.{decimals_per_coin(stake_currency)}f', - '.2f', 'd', 'd', 'd', 'd'] + '.2f', 'd', 's', 's'] def _get_line_header(first_column: str, stake_currency: str) -> List[str]: @@ -52,15 +52,26 @@ def _get_line_header(first_column: str, stake_currency: str) -> List[str]: """ return [first_column, 'Buys', 'Avg Profit %', 'Cum Profit %', f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration', - 'Wins', 'Draws', 'Losses'] + 'Win Draw Loss Win%'] -def _generate_result_line(result: DataFrame, max_open_trades: int, first_column: str) -> Dict: +def _generate_wins_draws_losses(wins, draws, losses): + if wins > 0 and losses == 0: + wl_ratio = '100' + elif wins == 0: + wl_ratio = '0' + else: + wl_ratio = f'{100.0 / (wins + draws + losses) * wins:.1f}' if losses > 0 else '100' + return f'{wins:>4} {draws:>4} {losses:>4} {wl_ratio:>4}' + + +def _generate_result_line(result: DataFrame, starting_balance: int, first_column: str) -> Dict: """ Generate one result dict, with "first_column" as key. """ profit_sum = result['profit_ratio'].sum() - profit_total = profit_sum / max_open_trades + # (end-capital - starting capital) / starting capital + profit_total = result['profit_abs'].sum() / starting_balance return { 'key': first_column, @@ -87,13 +98,13 @@ def _generate_result_line(result: DataFrame, max_open_trades: int, first_column: } -def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, max_open_trades: int, +def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, starting_balance: int, results: DataFrame, skip_nan: bool = False) -> List[Dict]: """ Generates and returns a list for the given backtest data and the results dataframe :param data: Dict of containing data that was used during backtesting. :param stake_currency: stake-currency - used to correctly name headers - :param max_open_trades: Maximum allowed open trades + :param starting_balance: Starting balance :param results: Dataframe containing the backtest results :param skip_nan: Print "left open" open trades :return: List of Dicts containing the metrics per pair @@ -106,10 +117,13 @@ def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, max_open_t if skip_nan and result['profit_abs'].isnull().all(): continue - tabular_data.append(_generate_result_line(result, max_open_trades, pair)) + tabular_data.append(_generate_result_line(result, starting_balance, pair)) + + # Sort by total profit %: + tabular_data = sorted(tabular_data, key=lambda k: k['profit_total_abs'], reverse=True) # Append Total - tabular_data.append(_generate_result_line(results, max_open_trades, 'TOTAL')) + tabular_data.append(_generate_result_line(results, starting_balance, 'TOTAL')) return tabular_data @@ -131,7 +145,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List tabular_data.append( { - 'sell_reason': reason.value, + 'sell_reason': reason, 'trades': count, 'wins': len(result[result['profit_abs'] > 0]), 'draws': len(result[result['profit_abs'] == 0]), @@ -148,7 +162,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List return tabular_data -def generate_strategy_metrics(all_results: Dict) -> List[Dict]: +def generate_strategy_comparison(all_results: Dict) -> List[Dict]: """ Generate summary per strategy :param all_results: Dict of containing results for all strategies @@ -158,8 +172,19 @@ def generate_strategy_metrics(all_results: Dict) -> List[Dict]: tabular_data = [] for strategy, results in all_results.items(): tabular_data.append(_generate_result_line( - results['results'], results['config']['max_open_trades'], strategy) + results['results'], results['config']['dry_run_wallet'], strategy) ) + try: + max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'], + value_col='profit_ratio') + max_drawdown_abs, _, _, _, _ = calculate_max_drawdown(results['results'], + value_col='profit_abs') + except ValueError: + max_drawdown_per = 0 + max_drawdown_abs = 0 + tabular_data[-1]['max_drawdown_per'] = round(max_drawdown_per * 100, 2) + tabular_data[-1]['max_drawdown_abs'] = \ + round_coin_value(max_drawdown_abs, results['config']['stake_currency'], False) return tabular_data @@ -189,43 +214,221 @@ def generate_edge_table(results: dict) -> str: floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore +def generate_trading_stats(results: DataFrame) -> Dict[str, Any]: + """ Generate overall trade statistics """ + if len(results) == 0: + return { + 'wins': 0, + 'losses': 0, + 'draws': 0, + 'holding_avg': timedelta(), + 'winner_holding_avg': timedelta(), + 'loser_holding_avg': timedelta(), + } + + winning_trades = results.loc[results['profit_ratio'] > 0] + draw_trades = results.loc[results['profit_ratio'] == 0] + losing_trades = results.loc[results['profit_ratio'] < 0] + zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) & + (results['sell_reason'] == 'trailing_stop_loss')]) + + holding_avg = (timedelta(minutes=round(results['trade_duration'].mean())) + if not results.empty else timedelta()) + winner_holding_avg = (timedelta(minutes=round(winning_trades['trade_duration'].mean())) + if not winning_trades.empty else timedelta()) + loser_holding_avg = (timedelta(minutes=round(losing_trades['trade_duration'].mean())) + if not losing_trades.empty else timedelta()) + + return { + 'wins': len(winning_trades), + 'losses': len(losing_trades), + 'draws': len(draw_trades), + 'holding_avg': holding_avg, + 'holding_avg_s': holding_avg.total_seconds(), + 'winner_holding_avg': winner_holding_avg, + 'winner_holding_avg_s': winner_holding_avg.total_seconds(), + 'loser_holding_avg': loser_holding_avg, + 'loser_holding_avg_s': loser_holding_avg.total_seconds(), + 'zero_duration_trades': zero_duration_trades, + } + + def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: + """ Generate daily statistics """ if len(results) == 0: return { 'backtest_best_day': 0, 'backtest_worst_day': 0, + 'backtest_best_day_abs': 0, + 'backtest_worst_day_abs': 0, 'winning_days': 0, 'draw_days': 0, 'losing_days': 0, - 'winner_holding_avg': timedelta(), - 'loser_holding_avg': timedelta(), } - daily_profit = results.resample('1d', on='close_date')['profit_ratio'].sum() + daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum() + daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10) + worst_rel = min(daily_profit_rel) + best_rel = max(daily_profit_rel) worst = min(daily_profit) best = max(daily_profit) winning_days = sum(daily_profit > 0) draw_days = sum(daily_profit == 0) losing_days = sum(daily_profit < 0) - winning_trades = results.loc[results['profit_ratio'] > 0] - losing_trades = results.loc[results['profit_ratio'] < 0] - return { - 'backtest_best_day': best, - 'backtest_worst_day': worst, + 'backtest_best_day': best_rel, + 'backtest_worst_day': worst_rel, + 'backtest_best_day_abs': best, + 'backtest_worst_day_abs': worst, 'winning_days': winning_days, 'draw_days': draw_days, 'losing_days': losing_days, - 'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean())) - if not winning_trades.empty else timedelta()), - 'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean())) - if not losing_trades.empty else timedelta()), } +def generate_strategy_stats(btdata: Dict[str, DataFrame], + strategy: str, + content: Dict[str, Any], + min_date: datetime, max_date: datetime, + market_change: float + ) -> Dict[str, Any]: + """ + :param btdata: Backtest data + :param strategy: Strategy name + :param content: Backtest result data in the format: + {'results: results, 'config: config}}. + :param min_date: Backtest start date + :param max_date: Backtest end date + :param market_change: float indicating the market change + :return: Dictionary containing results per strategy and a stratgy summary. + """ + results: Dict[str, DataFrame] = content['results'] + if not isinstance(results, DataFrame): + return {} + config = content['config'] + max_open_trades = min(config['max_open_trades'], len(btdata.keys())) + starting_balance = config['dry_run_wallet'] + stake_currency = config['stake_currency'] + + pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency, + starting_balance=starting_balance, + results=results, skip_nan=False) + sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades, + results=results) + left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency, + starting_balance=starting_balance, + results=results.loc[results['is_open']], + skip_nan=True) + daily_stats = generate_daily_stats(results) + trade_stats = generate_trading_stats(results) + best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'], + key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None + worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], + key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None + results['open_timestamp'] = results['open_date'].astype(int64) // 1e6 + results['close_timestamp'] = results['close_date'].astype(int64) // 1e6 + + backtest_days = (max_date - min_date).days + strat_stats = { + 'trades': results.to_dict(orient='records'), + 'locks': [lock.to_json() for lock in content['locks']], + 'best_pair': best_pair, + 'worst_pair': worst_pair, + 'results_per_pair': pair_results, + 'sell_reason_summary': sell_reason_stats, + 'left_open_trades': left_open_results, + 'total_trades': len(results), + 'total_volume': float(results['stake_amount'].sum()), + 'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0, + 'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0, + 'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0, + 'profit_total': results['profit_abs'].sum() / starting_balance, + 'profit_total_abs': results['profit_abs'].sum(), + 'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT), + 'backtest_start_ts': int(min_date.timestamp() * 1000), + 'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT), + 'backtest_end_ts': int(max_date.timestamp() * 1000), + 'backtest_days': backtest_days, + + 'backtest_run_start_ts': content['backtest_start_time'], + 'backtest_run_end_ts': content['backtest_end_time'], + + 'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0, + 'market_change': market_change, + 'pairlist': list(btdata.keys()), + 'stake_amount': config['stake_amount'], + 'stake_currency': config['stake_currency'], + 'stake_currency_decimals': decimals_per_coin(config['stake_currency']), + 'starting_balance': starting_balance, + 'dry_run_wallet': starting_balance, + 'final_balance': content['final_balance'], + 'rejected_signals': content['rejected_signals'], + 'max_open_trades': max_open_trades, + 'max_open_trades_setting': (config['max_open_trades'] + if config['max_open_trades'] != float('inf') else -1), + 'timeframe': config['timeframe'], + 'timerange': config.get('timerange', ''), + 'enable_protections': config.get('enable_protections', False), + 'strategy_name': strategy, + # Parameters relevant for backtesting + 'stoploss': config['stoploss'], + 'trailing_stop': config.get('trailing_stop', False), + 'trailing_stop_positive': config.get('trailing_stop_positive'), + 'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0), + 'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False), + 'use_custom_stoploss': config.get('use_custom_stoploss', False), + 'minimal_roi': config['minimal_roi'], + 'use_sell_signal': config['ask_strategy']['use_sell_signal'], + 'sell_profit_only': config['ask_strategy']['sell_profit_only'], + 'sell_profit_offset': config['ask_strategy']['sell_profit_offset'], + 'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'], + **daily_stats, + **trade_stats + } + + try: + max_drawdown, _, _, _, _ = calculate_max_drawdown( + results, value_col='profit_ratio') + drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown( + results, value_col='profit_abs') + strat_stats.update({ + 'max_drawdown': max_drawdown, + 'max_drawdown_abs': drawdown_abs, + 'drawdown_start': drawdown_start.strftime(DATETIME_PRINT_FORMAT), + 'drawdown_start_ts': drawdown_start.timestamp() * 1000, + 'drawdown_end': drawdown_end.strftime(DATETIME_PRINT_FORMAT), + 'drawdown_end_ts': drawdown_end.timestamp() * 1000, + + 'max_drawdown_low': low_val, + 'max_drawdown_high': high_val, + }) + + csum_min, csum_max = calculate_csum(results, starting_balance) + strat_stats.update({ + 'csum_min': csum_min, + 'csum_max': csum_max + }) + + except ValueError: + strat_stats.update({ + 'max_drawdown': 0.0, + 'max_drawdown_abs': 0.0, + 'max_drawdown_low': 0.0, + 'max_drawdown_high': 0.0, + 'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc), + 'drawdown_start_ts': 0, + 'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc), + 'drawdown_end_ts': 0, + 'csum_min': 0, + 'csum_max': 0 + }) + + return strat_stats + + def generate_backtest_stats(btdata: Dict[str, DataFrame], all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]], - min_date: Arrow, max_date: Arrow + min_date: datetime, max_date: datetime ) -> Dict[str, Any]: """ :param btdata: Backtest data @@ -233,107 +436,17 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], { Strategy: {'results: results, 'config: config}}. :param min_date: Backtest start date :param max_date: Backtest end date - :return: - Dictionary containing results per strategy and a stratgy summary. + :return: Dictionary containing results per strategy and a stratgy summary. """ result: Dict[str, Any] = {'strategy': {}} market_change = calculate_market_change(btdata, 'close') for strategy, content in all_results.items(): - results: Dict[str, DataFrame] = content['results'] - if not isinstance(results, DataFrame): - continue - config = content['config'] - max_open_trades = min(config['max_open_trades'], len(btdata.keys())) - stake_currency = config['stake_currency'] - - pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency, - max_open_trades=max_open_trades, - results=results, skip_nan=False) - sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades, - results=results) - left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency, - max_open_trades=max_open_trades, - results=results.loc[results['is_open']], - skip_nan=True) - daily_stats = generate_daily_stats(results) - best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'], - key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None - worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], - key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None - results['open_timestamp'] = results['open_date'].astype(int64) // 1e6 - results['close_timestamp'] = results['close_date'].astype(int64) // 1e6 - - backtest_days = (max_date - min_date).days - strat_stats = { - 'trades': results.to_dict(orient='records'), - 'locks': [lock.to_json() for lock in content['locks']], - 'best_pair': best_pair, - 'worst_pair': worst_pair, - 'results_per_pair': pair_results, - 'sell_reason_summary': sell_reason_stats, - 'left_open_trades': left_open_results, - 'total_trades': len(results), - 'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0, - 'profit_total': results['profit_ratio'].sum() / max_open_trades, - 'profit_total_abs': results['profit_abs'].sum(), - 'backtest_start': min_date.datetime, - 'backtest_start_ts': min_date.int_timestamp * 1000, - 'backtest_end': max_date.datetime, - 'backtest_end_ts': max_date.int_timestamp * 1000, - 'backtest_days': backtest_days, - - 'backtest_run_start_ts': content['backtest_start_time'], - 'backtest_run_end_ts': content['backtest_end_time'], - - 'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0, - 'market_change': market_change, - 'pairlist': list(btdata.keys()), - 'stake_amount': config['stake_amount'], - 'stake_currency': config['stake_currency'], - 'max_open_trades': max_open_trades, - 'max_open_trades_setting': (config['max_open_trades'] - if config['max_open_trades'] != float('inf') else -1), - 'timeframe': config['timeframe'], - 'timerange': config.get('timerange', ''), - 'enable_protections': config.get('enable_protections', False), - 'strategy_name': strategy, - # Parameters relevant for backtesting - 'stoploss': config['stoploss'], - 'trailing_stop': config.get('trailing_stop', False), - 'trailing_stop_positive': config.get('trailing_stop_positive'), - 'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0), - 'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False), - 'use_custom_stoploss': config.get('use_custom_stoploss', False), - 'minimal_roi': config['minimal_roi'], - 'use_sell_signal': config['ask_strategy']['use_sell_signal'], - 'sell_profit_only': config['ask_strategy']['sell_profit_only'], - 'sell_profit_offset': config['ask_strategy']['sell_profit_offset'], - 'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'], - **daily_stats, - } + strat_stats = generate_strategy_stats(btdata, strategy, content, + min_date, max_date, market_change=market_change) result['strategy'][strategy] = strat_stats - try: - max_drawdown, drawdown_start, drawdown_end = calculate_max_drawdown( - results, value_col='profit_ratio') - strat_stats.update({ - 'max_drawdown': max_drawdown, - 'drawdown_start': drawdown_start, - 'drawdown_start_ts': drawdown_start.timestamp() * 1000, - 'drawdown_end': drawdown_end, - 'drawdown_end_ts': drawdown_end.timestamp() * 1000, - }) - except ValueError: - strat_stats.update({ - 'max_drawdown': 0.0, - 'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc), - 'drawdown_start_ts': 0, - 'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc), - 'drawdown_end_ts': 0, - }) - - strategy_results = generate_strategy_metrics(all_results=all_results) + strategy_results = generate_strategy_comparison(all_results=all_results) result['strategy_comparison'] = strategy_results @@ -356,7 +469,8 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st floatfmt = _get_line_floatfmt(stake_currency) output = [[ t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], - t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses'] + t['profit_total_pct'], t['duration_avg'], + _generate_wins_draws_losses(t['wins'], t['draws'], t['losses']) ] for t in pair_results] # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(output, headers=headers, @@ -373,9 +487,7 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren headers = [ 'Sell Reason', 'Sells', - 'Wins', - 'Draws', - 'Losses', + 'Win Draws Loss Win%', 'Avg Profit %', 'Cum Profit %', f'Tot Profit {stake_currency}', @@ -383,7 +495,8 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren ] output = [[ - t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'], + t['sell_reason'], t['trades'], + _generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), t['profit_mean_pct'], t['profit_sum_pct'], round_coin_value(t['profit_total_abs'], stake_currency, False), t['profit_total_pct'], @@ -401,11 +514,22 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str: """ floatfmt = _get_line_floatfmt(stake_currency) headers = _get_line_header('Strategy', stake_currency) + # _get_line_header() is also used for per-pair summary. Per-pair drawdown is mostly useless + # therefore we slip this column in only for strategy summary here. + headers.append('Drawdown') + + # Align drawdown string on the center two space separator. + drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results] + dd_pad_abs = max([len(t['max_drawdown_abs']) for t in strategy_results]) + dd_pad_per = max([len(dd) for dd in drawdown]) + drawdown = [f'{t["max_drawdown_abs"]:>{dd_pad_abs}} {stake_currency} {dd:>{dd_pad_per}}%' + for t, dd in zip(strategy_results, drawdown)] output = [[ t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], - t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses'] - ] for t in strategy_results] + t['profit_total_pct'], t['duration_avg'], + _generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), drawdown] + for t, drawdown in zip(strategy_results, drawdown)] # Ignore type as floatfmt does allow tuples but mypy does not know that return tabulate(output, headers=headers, floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") @@ -415,14 +539,36 @@ def text_table_add_metrics(strat_results: Dict) -> str: if len(strat_results['trades']) > 0: best_trade = max(strat_results['trades'], key=lambda x: x['profit_ratio']) worst_trade = min(strat_results['trades'], key=lambda x: x['profit_ratio']) + + # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show + # command stores these results and newer version of freqtrade must be able to handle old + # results with missing new fields. + zero_duration_trades = '--' + + if 'zero_duration_trades' in strat_results: + zero_duration_trades_per = \ + 100.0 / strat_results['total_trades'] * strat_results['zero_duration_trades'] + zero_duration_trades = f'{zero_duration_trades_per:.2f}% ' \ + f'({strat_results["zero_duration_trades"]})' + metrics = [ - ('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)), - ('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)), + ('Backtesting from', strat_results['backtest_start']), + ('Backtesting to', strat_results['backtest_end']), ('Max open trades', strat_results['max_open_trades']), ('', ''), # Empty line to improve readability ('Total trades', strat_results['total_trades']), - ('Total Profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"), + ('Starting balance', round_coin_value(strat_results['starting_balance'], + strat_results['stake_currency'])), + ('Final balance', round_coin_value(strat_results['final_balance'], + strat_results['stake_currency'])), + ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], + strat_results['stake_currency'])), + ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"), ('Trades per day', strat_results['trades_per_day']), + ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'], + strat_results['stake_currency'])), + ('Total trade volume', round_coin_value(strat_results['total_volume'], + strat_results['stake_currency'])), ('', ''), # Empty line to improve readability ('Best Pair', f"{strat_results['best_pair']['key']} " f"{round(strat_results['best_pair']['profit_sum_pct'], 2)}%"), @@ -432,55 +578,87 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Worst trade', f"{worst_trade['pair']} " f"{round(worst_trade['profit_ratio'] * 100, 2)}%"), - ('Best day', f"{round(strat_results['backtest_best_day'] * 100, 2)}%"), - ('Worst day', f"{round(strat_results['backtest_worst_day'] * 100, 2)}%"), + ('Best day', round_coin_value(strat_results['backtest_best_day_abs'], + strat_results['stake_currency'])), + ('Worst day', round_coin_value(strat_results['backtest_worst_day_abs'], + strat_results['stake_currency'])), ('Days win/draw/lose', f"{strat_results['winning_days']} / " f"{strat_results['draw_days']} / {strat_results['losing_days']}"), ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"), + ('Zero Duration Trades', zero_duration_trades), + ('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')), ('', ''), # Empty line to improve readability - ('Max Drawdown', f"{round(strat_results['max_drawdown'] * 100, 2)}%"), - ('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)), - ('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)), + + ('Min balance', round_coin_value(strat_results['csum_min'], + strat_results['stake_currency'])), + ('Max balance', round_coin_value(strat_results['csum_max'], + strat_results['stake_currency'])), + + ('Drawdown', f"{round(strat_results['max_drawdown'] * 100, 2)}%"), + ('Drawdown', round_coin_value(strat_results['max_drawdown_abs'], + strat_results['stake_currency'])), + ('Drawdown high', round_coin_value(strat_results['max_drawdown_high'], + strat_results['stake_currency'])), + ('Drawdown low', round_coin_value(strat_results['max_drawdown_low'], + strat_results['stake_currency'])), + ('Drawdown Start', strat_results['drawdown_start']), + ('Drawdown End', strat_results['drawdown_end']), ('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"), ] return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl") else: - return '' + start_balance = round_coin_value(strat_results['starting_balance'], + strat_results['stake_currency']) + stake_amount = round_coin_value( + strat_results['stake_amount'], strat_results['stake_currency'] + ) if strat_results['stake_amount'] != UNLIMITED_STAKE_AMOUNT else 'unlimited' + + message = ("No trades made. " + f"Your starting balance was {start_balance}, " + f"and your stake was {stake_amount}." + ) + return message + + +def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str): + """ + Print results for one strategy + """ + # Print results + print(f"Result for strategy {strategy}") + table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency) + if isinstance(table, str): + print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + + table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'], + stake_currency=stake_currency) + if isinstance(table, str) and len(table) > 0: + print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '=')) + print(table) + + table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency) + if isinstance(table, str) and len(table) > 0: + print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + + table = text_table_add_metrics(results) + if isinstance(table, str) and len(table) > 0: + print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '=')) + print(table) + + if isinstance(table, str) and len(table) > 0: + print('=' * len(table.splitlines()[0])) + print() def show_backtest_results(config: Dict, backtest_stats: Dict): stake_currency = config['stake_currency'] for strategy, results in backtest_stats['strategy'].items(): - - # Print results - print(f"Result for strategy {strategy}") - table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency) - if isinstance(table, str): - print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) - print(table) - - table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'], - stake_currency=stake_currency) - if isinstance(table, str) and len(table) > 0: - print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '=')) - print(table) - - table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency) - if isinstance(table, str) and len(table) > 0: - print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) - print(table) - - table = text_table_add_metrics(results) - if isinstance(table, str) and len(table) > 0: - print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '=')) - print(table) - - if isinstance(table, str) and len(table) > 0: - print('=' * len(table.splitlines()[0])) - print() + show_backtest_result(strategy, results, stake_currency) if len(backtest_stats['strategy']) > 1: # Print Strategy summary table diff --git a/freqtrade/optimize/space/__init__.py b/freqtrade/optimize/space/__init__.py new file mode 100644 index 000000000..bbdac4ab9 --- /dev/null +++ b/freqtrade/optimize/space/__init__.py @@ -0,0 +1,4 @@ +# flake8: noqa: F401 +from skopt.space import Categorical, Dimension, Integer, Real + +from .decimalspace import SKDecimal diff --git a/freqtrade/optimize/space/decimalspace.py b/freqtrade/optimize/space/decimalspace.py new file mode 100644 index 000000000..643999cc1 --- /dev/null +++ b/freqtrade/optimize/space/decimalspace.py @@ -0,0 +1,33 @@ +import numpy as np +from skopt.space import Integer + + +class SKDecimal(Integer): + + def __init__(self, low, high, decimals=3, prior="uniform", base=10, transform=None, + name=None, dtype=np.int64): + self.decimals = decimals + _low = int(low * pow(10, self.decimals)) + _high = int(high * pow(10, self.decimals)) + # trunc to precision to avoid points out of space + self.low_orig = round(_low * pow(0.1, self.decimals), self.decimals) + self.high_orig = round(_high * pow(0.1, self.decimals), self.decimals) + + super().__init__(_low, _high, prior, base, transform, name, dtype) + + def __repr__(self): + return "Decimal(low={}, high={}, decimals={}, prior='{}', transform='{}')".format( + self.low_orig, self.high_orig, self.decimals, self.prior, self.transform_) + + def __contains__(self, point): + if isinstance(point, list): + point = np.array(point) + return self.low_orig <= point <= self.high_orig + + def transform(self, Xt): + aa = [int(x * pow(10, self.decimals)) for x in Xt] + return super().transform(aa) + + def inverse_transform(self, Xt): + res = super().inverse_transform(Xt) + return [round(x * pow(0.1, self.decimals), self.decimals) for x in res] diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py index 35f2bc406..d1fcac0ba 100644 --- a/freqtrade/persistence/__init__.py +++ b/freqtrade/persistence/__init__.py @@ -1,4 +1,5 @@ # flake8: noqa: F401 -from freqtrade.persistence.models import Order, Trade, clean_dry_run_db, cleanup_db, init_db +from freqtrade.persistence.models import (LocalTrade, Order, Trade, clean_dry_run_db, cleanup_db, + init_db) from freqtrade.persistence.pairlock_middleware import PairLocks diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index ed976c2a9..00c9b91eb 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -1,7 +1,7 @@ import logging from typing import List -from sqlalchemy import inspect +from sqlalchemy import inspect, text logger = logging.getLogger(__name__) @@ -62,15 +62,17 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col amount_requested = get_column_def(cols, 'amount_requested', 'amount') # Schema migration necessary - engine.execute(f"alter table trades rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") + with engine.begin() as connection: + connection.execute(text(f"alter table trades rename to {table_back_name}")) + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + connection.execute(text(f"drop index {index['name']}")) # let SQLAlchemy create the schema as required decl_base.metadata.create_all(engine) # Copy data back - following the correct schema - engine.execute(f"""insert into trades + with engine.begin() as connection: + connection.execute(text(f"""insert into trades (id, exchange, pair, is_open, fee_open, fee_open_cost, fee_open_currency, fee_close, fee_close_cost, fee_open_currency, open_rate, @@ -104,11 +106,12 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col {strategy} strategy, {timeframe} timeframe, {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs from {table_back_name} - """) + """)) def migrate_open_orders_to_trades(engine): - engine.execute(""" + with engine.begin() as connection: + connection.execute(text(""" insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open) select id ft_trade_id, pair ft_pair, open_order_id, case when close_rate_requested is null then 'buy' @@ -120,7 +123,30 @@ def migrate_open_orders_to_trades(engine): 'stoploss' ft_order_side, 1 ft_is_open from trades where stoploss_order_id is not null - """) + """)) + + +def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List): + # Schema migration necessary + + with engine.begin() as connection: + connection.execute(text(f"alter table orders rename to {table_back_name}")) + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + connection.execute(text(f"drop index {index['name']}")) + + # let SQLAlchemy create the schema as required + decl_base.metadata.create_all(engine) + with engine.begin() as connection: + connection.execute(text(f""" + insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, + status, symbol, order_type, side, price, amount, filled, average, remaining, cost, + order_date, order_filled_date, order_update_date) + select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, + status, symbol, order_type, side, price, amount, filled, null average, remaining, cost, + order_date, order_filled_date, order_update_date + from {table_back_name} + """)) def check_migrate(engine, decl_base, previous_tables) -> None: @@ -141,10 +167,15 @@ def check_migrate(engine, decl_base, previous_tables) -> None: inspector = inspect(engine) cols = inspector.get_columns('trades') - if 'orders' not in previous_tables: + if 'orders' not in previous_tables and 'trades' in previous_tables: logger.info('Moving open orders to Orders table.') migrate_open_orders_to_trades(engine) else: - pass - # Empty for now - as there is only one iteration of the orders table so far. - # table_back_name = get_backup_name(tabs, 'orders_bak') + cols_order = inspector.get_columns('orders') + + if not has_column(cols_order, 'average'): + tabs = get_table_names_for_table(inspector, 'orders') + # Empty for now - as there is only one iteration of the orders table so far. + table_back_name = get_backup_name(tabs, 'orders_bak') + + migrate_orders_table(decl_base, inspector, engine, table_back_name, cols) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index dff59819c..bbbc55c13 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -6,18 +6,15 @@ from datetime import datetime, timezone from decimal import Decimal from typing import Any, Dict, List, Optional -import arrow from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Query, relationship -from sqlalchemy.orm.scoping import scoped_session -from sqlalchemy.orm.session import sessionmaker +from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker from sqlalchemy.pool import StaticPool from sqlalchemy.sql.schema import UniqueConstraint from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.enums import SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.misc import safe_value_fallback from freqtrade.persistence.migrations import check_migrate @@ -42,16 +39,18 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None: """ kwargs = {} - # Take care of thread ownership if in-memory db if db_url == 'sqlite://': kwargs.update({ - 'connect_args': {'check_same_thread': False}, 'poolclass': StaticPool, - 'echo': False, + }) + # Take care of thread ownership + if db_url.startswith('sqlite://'): + kwargs.update({ + 'connect_args': {'check_same_thread': False}, }) try: - engine = create_engine(db_url, **kwargs) + engine = create_engine(db_url, future=True, **kwargs) except NoSuchModuleError: raise OperationalException(f"Given value for db_url: '{db_url}' " f"is no valid database URL! (See {_SQL_DOCS_URL})") @@ -59,13 +58,10 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None: # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope # Scoped sessions proxy requests to the appropriate thread-local session. # We should use the scoped_session object - not a seperately initialized version - Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) - Trade.query = Trade.session.query_property() - # Copy session attributes to order object too - Order.session = Trade.session - Order.query = Order.session.query_property() - PairLock.session = Trade.session - PairLock.query = PairLock.session.query_property() + Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True)) + Trade.query = Trade._session.query_property() + Order.query = Trade._session.query_property() + PairLock.query = Trade._session.query_property() previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) @@ -81,7 +77,7 @@ def cleanup_db() -> None: Flushes all pending operations to disk. :return: None """ - Trade.session.flush() + Trade.commit() def clean_dry_run_db() -> None: @@ -93,6 +89,7 @@ def clean_dry_run_db() -> None: # Check we are updating only a dry_run order not a prod one if 'dry_run' in trade.open_order_id: trade.open_order_id = None + Trade.commit() class Order(_DECL_BASE): @@ -116,16 +113,17 @@ class Order(_DECL_BASE): trade = relationship("Trade", back_populates="orders") - ft_order_side = Column(String, nullable=False) - ft_pair = Column(String, nullable=False) + ft_order_side = Column(String(25), nullable=False) + ft_pair = Column(String(25), nullable=False) ft_is_open = Column(Boolean, nullable=False, default=True, index=True) - order_id = Column(String, nullable=False, index=True) - status = Column(String, nullable=True) - symbol = Column(String, nullable=True) - order_type = Column(String, nullable=True) - side = Column(String, nullable=True) + order_id = Column(String(255), nullable=False, index=True) + status = Column(String(255), nullable=True) + symbol = Column(String(25), nullable=True) + order_type = Column(String(50), nullable=True) + side = Column(String(25), nullable=True) price = Column(Float, nullable=True) + average = Column(Float, nullable=True) amount = Column(Float, nullable=True) filled = Column(Float, nullable=True) remaining = Column(Float, nullable=True) @@ -154,6 +152,7 @@ class Order(_DECL_BASE): self.price = order.get('price', self.price) self.amount = order.get('amount', self.amount) self.filled = order.get('filled', self.filled) + self.average = order.get('average', self.average) self.remaining = order.get('remaining', self.remaining) self.cost = order.get('cost', self.cost) if 'timestamp' in order and order['timestamp'] is not None: @@ -163,8 +162,8 @@ class Order(_DECL_BASE): if self.status in ('closed', 'canceled', 'cancelled'): self.ft_is_open = False if order.get('filled', 0) > 0: - self.order_filled_date = arrow.utcnow().datetime - self.order_update_date = arrow.utcnow().datetime + self.order_filled_date = datetime.now(timezone.utc) + self.order_update_date = datetime.now(timezone.utc) @staticmethod def update_orders(orders: List['Order'], order: Dict[str, Any]): @@ -179,6 +178,7 @@ class Order(_DECL_BASE): if filtered_orders: oobj = filtered_orders[0] oobj.update_from_ccxt_object(order) + Order.query.session.commit() else: logger.warning(f"Did not find order for {order}.") @@ -199,67 +199,69 @@ class Order(_DECL_BASE): return Order.query.filter(Order.ft_is_open.is_(True)).all() -class Trade(_DECL_BASE): +class LocalTrade(): """ Trade database model. - Also handles updating and querying trades + Used in backtesting - must be aligned to Trade model! + """ - __tablename__ = 'trades' - - use_db: bool = True + use_db: bool = False # Trades container for backtesting - trades: List['Trade'] = [] + trades: List['LocalTrade'] = [] + trades_open: List['LocalTrade'] = [] + total_profit: float = 0 - id = Column(Integer, primary_key=True) + id: int = 0 - orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan") + orders: List[Order] = [] - exchange = Column(String, nullable=False) - pair = Column(String, nullable=False, index=True) - is_open = Column(Boolean, nullable=False, default=True, index=True) - fee_open = Column(Float, nullable=False, default=0.0) - fee_open_cost = Column(Float, nullable=True) - fee_open_currency = Column(String, nullable=True) - fee_close = Column(Float, nullable=False, default=0.0) - fee_close_cost = Column(Float, nullable=True) - fee_close_currency = Column(String, nullable=True) - open_rate = Column(Float) - open_rate_requested = Column(Float) + exchange: str = '' + pair: str = '' + is_open: bool = True + fee_open: float = 0.0 + fee_open_cost: Optional[float] = None + fee_open_currency: str = '' + fee_close: float = 0.0 + fee_close_cost: Optional[float] = None + fee_close_currency: str = '' + open_rate: float = 0.0 + open_rate_requested: Optional[float] = None # open_trade_value - calculated via _calc_open_trade_value - open_trade_value = Column(Float) - close_rate = Column(Float) - close_rate_requested = Column(Float) - close_profit = Column(Float) - close_profit_abs = Column(Float) - stake_amount = Column(Float, nullable=False) - amount = Column(Float) - amount_requested = Column(Float) - open_date = Column(DateTime, nullable=False, default=datetime.utcnow) - close_date = Column(DateTime) - open_order_id = Column(String) + open_trade_value: float = 0.0 + close_rate: Optional[float] = None + close_rate_requested: Optional[float] = None + close_profit: Optional[float] = None + close_profit_abs: Optional[float] = None + stake_amount: float = 0.0 + amount: float = 0.0 + amount_requested: Optional[float] = None + open_date: datetime + close_date: Optional[datetime] = None + open_order_id: Optional[str] = None # absolute value of the stop loss - stop_loss = Column(Float, nullable=True, default=0.0) + stop_loss: float = 0.0 # percentage value of the stop loss - stop_loss_pct = Column(Float, nullable=True) + stop_loss_pct: float = 0.0 # absolute value of the initial stop loss - initial_stop_loss = Column(Float, nullable=True, default=0.0) + initial_stop_loss: float = 0.0 # percentage value of the initial stop loss - initial_stop_loss_pct = Column(Float, nullable=True) + initial_stop_loss_pct: float = 0.0 # stoploss order id which is on exchange - stoploss_order_id = Column(String, nullable=True, index=True) + stoploss_order_id: Optional[str] = None # last update time of the stoploss order on exchange - stoploss_last_update = Column(DateTime, nullable=True) + stoploss_last_update: Optional[datetime] = None # absolute value of the highest reached price - max_rate = Column(Float, nullable=True, default=0.0) + max_rate: float = 0.0 # Lowest price reached - min_rate = Column(Float, nullable=True) - sell_reason = Column(String, nullable=True) - sell_order_status = Column(String, nullable=True) - strategy = Column(String, nullable=True) - timeframe = Column(Integer, nullable=True) + min_rate: float = 0.0 + sell_reason: str = '' + sell_order_status: str = '' + strategy: str = '' + timeframe: Optional[int] = None def __init__(self, **kwargs): - super().__init__(**kwargs) + for key in kwargs: + setattr(self, key, kwargs[key]) self.recalc_open_trade_value() def __repr__(self): @@ -268,6 +270,14 @@ class Trade(_DECL_BASE): return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' f'open_rate={self.open_rate:.8f}, open_since={open_since})') + @property + def open_date_utc(self): + return self.open_date.replace(tzinfo=timezone.utc) + + @property + def close_date_utc(self): + return self.close_date.replace(tzinfo=timezone.utc) + def to_json(self) -> Dict[str, Any]: return { 'trade_id': self.id, @@ -287,15 +297,12 @@ class Trade(_DECL_BASE): 'fee_close_cost': self.fee_close_cost, 'fee_close_currency': self.fee_close_currency, - 'open_date_hum': arrow.get(self.open_date).humanize(), 'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT), 'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000), 'open_rate': self.open_rate, 'open_rate_requested': self.open_rate_requested, 'open_trade_value': round(self.open_trade_value, 8), - 'close_date_hum': (arrow.get(self.close_date).humanize() - if self.close_date else None), 'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT) if self.close_date else None), 'close_timestamp': int(self.close_date.replace( @@ -306,9 +313,9 @@ class Trade(_DECL_BASE): 'close_profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None, 'close_profit_abs': self.close_profit_abs, # Deprecated - 'trade_duration_s': (int((self.close_date - self.open_date).total_seconds()) + 'trade_duration_s': (int((self.close_date_utc - self.open_date_utc).total_seconds()) if self.close_date else None), - 'trade_duration': (int((self.close_date - self.open_date).total_seconds() // 60) + 'trade_duration': (int((self.close_date_utc - self.open_date_utc).total_seconds() // 60) if self.close_date else None), 'profit_ratio': self.close_profit, @@ -341,8 +348,9 @@ class Trade(_DECL_BASE): """ Resets all trades. Only active for backtesting mode. """ - if not Trade.use_db: - Trade.trades = [] + LocalTrade.trades = [] + LocalTrade.trades_open = [] + LocalTrade.total_profit = 0 def adjust_min_max_rates(self, current_price: float) -> None: """ @@ -410,8 +418,8 @@ class Trade(_DECL_BASE): if order_type in ('market', 'limit') and order['side'] == 'buy': # Update open rate and actual amount - self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price')) - self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount')) + self.open_rate = float(safe_value_fallback(order, 'average', 'price')) + self.amount = float(safe_value_fallback(order, 'filled', 'amount')) self.recalc_open_trade_value() if self.is_open: logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.') @@ -423,19 +431,20 @@ class Trade(_DECL_BASE): elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss + self.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value if self.is_open: logger.info(f'{order_type.upper()} is hit for {self}.') - self.close(order['average']) + self.close(safe_value_fallback(order, 'average', 'price')) else: raise ValueError(f'Unknown order type: {order_type}') - cleanup_db() + Trade.commit() def close(self, rate: float, *, show_msg: bool = True) -> None: """ Sets close_rate to the given rate, calculates total profit and marks trade as closed """ - self.close_rate = Decimal(rate) + self.close_rate = rate self.close_profit = self.calc_profit_ratio() self.close_profit_abs = self.calc_profit() self.close_date = self.close_date or datetime.utcnow() @@ -480,14 +489,6 @@ class Trade(_DECL_BASE): def update_order(self, order: Dict) -> None: Order.update_orders(self.orders, order) - def delete(self) -> None: - - for order in self.orders: - Order.session.delete(order) - - Trade.session.delete(self) - Trade.session.flush() - def _calc_open_trade_value(self) -> float: """ Calculate the open_rate including open_fee. @@ -517,7 +518,7 @@ class Trade(_DECL_BASE): if rate is None and not self.close_rate: return 0.0 - sell_trade = Decimal(self.amount) * Decimal(rate or self.close_rate) + sell_trade = Decimal(self.amount) * Decimal(rate or self.close_rate) # type: ignore fees = sell_trade * Decimal(fee or self.fee_close) return float(sell_trade - fees) @@ -551,6 +552,8 @@ class Trade(_DECL_BASE): rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) + if self.open_trade_value == 0.0: + return 0.0 profit_ratio = (close_trade_value / self.open_trade_value) - 1 return float(f"{profit_ratio:.8f}") @@ -569,27 +572,10 @@ class Trade(_DECL_BASE): else: return None - @staticmethod - def get_trades(trade_filter=None) -> Query: - """ - Helper function to query Trades using filters. - :param trade_filter: Optional filter to apply to trades - Can be either a Filter object, or a List of filters - e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])` - e.g. `(trade_filter=Trade.id == trade_id)` - :return: unsorted query object - """ - if trade_filter is not None: - if not isinstance(trade_filter, list): - trade_filter = [trade_filter] - return Trade.query.filter(*trade_filter) - else: - return Trade.query - @staticmethod def get_trades_proxy(*, pair: str = None, is_open: bool = None, open_date: datetime = None, close_date: datetime = None, - ) -> List['Trade']: + ) -> List['LocalTrade']: """ Helper function to query Trades. Returns a List of trades, filtered on the parameters given. @@ -598,110 +584,47 @@ class Trade(_DECL_BASE): :return: unsorted List[Trade] """ - if Trade.use_db: - trade_filter = [] - if pair: - trade_filter.append(Trade.pair == pair) - if open_date: - trade_filter.append(Trade.open_date > open_date) - if close_date: - trade_filter.append(Trade.close_date > close_date) - if is_open is not None: - trade_filter.append(Trade.is_open.is_(is_open)) - return Trade.get_trades(trade_filter).all() + + # Offline mode - without database + if is_open is not None: + if is_open: + sel_trades = LocalTrade.trades_open + else: + sel_trades = LocalTrade.trades + else: - # Offline mode - without database - sel_trades = [trade for trade in Trade.trades] - if pair: - sel_trades = [trade for trade in sel_trades if trade.pair == pair] - if open_date: - sel_trades = [trade for trade in sel_trades if trade.open_date > open_date] - if close_date: - sel_trades = [trade for trade in sel_trades if trade.close_date - and trade.close_date > close_date] - if is_open is not None: - sel_trades = [trade for trade in sel_trades if trade.is_open == is_open] - return sel_trades + # Not used during backtesting, but might be used by a strategy + sel_trades = list(LocalTrade.trades + LocalTrade.trades_open) + + if pair: + sel_trades = [trade for trade in sel_trades if trade.pair == pair] + if open_date: + sel_trades = [trade for trade in sel_trades if trade.open_date > open_date] + if close_date: + sel_trades = [trade for trade in sel_trades if trade.close_date + and trade.close_date > close_date] + + return sel_trades + + @staticmethod + def close_bt_trade(trade): + LocalTrade.trades_open.remove(trade) + LocalTrade.trades.append(trade) + LocalTrade.total_profit += trade.close_profit_abs + + @staticmethod + def add_bt_trade(trade): + if trade.is_open: + LocalTrade.trades_open.append(trade) + else: + LocalTrade.trades.append(trade) @staticmethod def get_open_trades() -> List[Any]: """ Query trades from persistence layer """ - return Trade.get_trades(Trade.is_open.is_(True)).all() - - @staticmethod - def get_open_order_trades(): - """ - Returns all open trades - """ - return Trade.get_trades(Trade.open_order_id.isnot(None)).all() - - @staticmethod - def get_open_trades_without_assigned_fees(): - """ - Returns all open trades which don't have open fees set correctly - """ - return Trade.get_trades([Trade.fee_open_currency.is_(None), - Trade.orders.any(), - Trade.is_open.is_(True), - ]).all() - - @staticmethod - def get_sold_trades_without_assigned_fees(): - """ - Returns all closed trades which don't have fees set correctly - """ - return Trade.get_trades([Trade.fee_close_currency.is_(None), - Trade.orders.any(), - Trade.is_open.is_(False), - ]).all() - - @staticmethod - def total_open_trades_stakes() -> float: - """ - Calculates total invested amount in open trades - in stake currency - """ - total_open_stake_amount = Trade.session.query(func.sum(Trade.stake_amount))\ - .filter(Trade.is_open.is_(True))\ - .scalar() - return total_open_stake_amount or 0 - - @staticmethod - def get_overall_performance() -> List[Dict[str, Any]]: - """ - Returns List of dicts containing all Trades, including profit and trade count - """ - pair_rates = Trade.session.query( - Trade.pair, - func.sum(Trade.close_profit).label('profit_sum'), - func.count(Trade.pair).label('count') - ).filter(Trade.is_open.is_(False))\ - .group_by(Trade.pair) \ - .order_by(desc('profit_sum')) \ - .all() - return [ - { - 'pair': pair, - 'profit': rate, - 'count': count - } - for pair, rate, count in pair_rates - ] - - @staticmethod - def get_best_pair(): - """ - Get best pair with closed trade. - :returns: Tuple containing (pair, profit_sum) - """ - best_pair = Trade.session.query( - Trade.pair, func.sum(Trade.close_profit).label('profit_sum') - ).filter(Trade.is_open.is_(False)) \ - .group_by(Trade.pair) \ - .order_by(desc('profit_sum')).first() - return best_pair + return Trade.get_trades_proxy(is_open=True) @staticmethod def stoploss_reinitialization(desired_stoploss): @@ -723,6 +646,215 @@ class Trade(_DECL_BASE): logger.info(f"New stoploss: {trade.stop_loss}.") +class Trade(_DECL_BASE, LocalTrade): + """ + Trade database model. + Also handles updating and querying trades + + Note: Fields must be aligned with LocalTrade class + """ + __tablename__ = 'trades' + + use_db: bool = True + + id = Column(Integer, primary_key=True) + + orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan") + + exchange = Column(String(25), nullable=False) + pair = Column(String(25), nullable=False, index=True) + is_open = Column(Boolean, nullable=False, default=True, index=True) + fee_open = Column(Float, nullable=False, default=0.0) + fee_open_cost = Column(Float, nullable=True) + fee_open_currency = Column(String(25), nullable=True) + fee_close = Column(Float, nullable=False, default=0.0) + fee_close_cost = Column(Float, nullable=True) + fee_close_currency = Column(String(25), nullable=True) + open_rate = Column(Float) + open_rate_requested = Column(Float) + # open_trade_value - calculated via _calc_open_trade_value + open_trade_value = Column(Float) + close_rate = Column(Float) + close_rate_requested = Column(Float) + close_profit = Column(Float) + close_profit_abs = Column(Float) + stake_amount = Column(Float, nullable=False) + amount = Column(Float) + amount_requested = Column(Float) + open_date = Column(DateTime, nullable=False, default=datetime.utcnow) + close_date = Column(DateTime) + open_order_id = Column(String(255)) + # absolute value of the stop loss + stop_loss = Column(Float, nullable=True, default=0.0) + # percentage value of the stop loss + stop_loss_pct = Column(Float, nullable=True) + # absolute value of the initial stop loss + initial_stop_loss = Column(Float, nullable=True, default=0.0) + # percentage value of the initial stop loss + initial_stop_loss_pct = Column(Float, nullable=True) + # stoploss order id which is on exchange + stoploss_order_id = Column(String(255), nullable=True, index=True) + # last update time of the stoploss order on exchange + stoploss_last_update = Column(DateTime, nullable=True) + # absolute value of the highest reached price + max_rate = Column(Float, nullable=True, default=0.0) + # Lowest price reached + min_rate = Column(Float, nullable=True) + sell_reason = Column(String(100), nullable=True) + sell_order_status = Column(String(100), nullable=True) + strategy = Column(String(100), nullable=True) + timeframe = Column(Integer, nullable=True) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.recalc_open_trade_value() + + def delete(self) -> None: + + for order in self.orders: + Order.query.session.delete(order) + + Trade.query.session.delete(self) + Trade.commit() + + @staticmethod + def commit(): + Trade.query.session.commit() + + @staticmethod + def get_trades_proxy(*, pair: str = None, is_open: bool = None, + open_date: datetime = None, close_date: datetime = None, + ) -> List['LocalTrade']: + """ + Helper function to query Trades.j + Returns a List of trades, filtered on the parameters given. + In live mode, converts the filter to a database query and returns all rows + In Backtest mode, uses filters on Trade.trades to get the result. + + :return: unsorted List[Trade] + """ + if Trade.use_db: + trade_filter = [] + if pair: + trade_filter.append(Trade.pair == pair) + if open_date: + trade_filter.append(Trade.open_date > open_date) + if close_date: + trade_filter.append(Trade.close_date > close_date) + if is_open is not None: + trade_filter.append(Trade.is_open.is_(is_open)) + return Trade.get_trades(trade_filter).all() + else: + return LocalTrade.get_trades_proxy( + pair=pair, is_open=is_open, + open_date=open_date, + close_date=close_date + ) + + @staticmethod + def get_trades(trade_filter=None) -> Query: + """ + Helper function to query Trades using filters. + NOTE: Not supported in Backtesting. + :param trade_filter: Optional filter to apply to trades + Can be either a Filter object, or a List of filters + e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])` + e.g. `(trade_filter=Trade.id == trade_id)` + :return: unsorted query object + """ + if not Trade.use_db: + raise NotImplementedError('`Trade.get_trades()` not supported in backtesting mode.') + if trade_filter is not None: + if not isinstance(trade_filter, list): + trade_filter = [trade_filter] + return Trade.query.filter(*trade_filter) + else: + return Trade.query + + @staticmethod + def get_open_order_trades(): + """ + Returns all open trades + NOTE: Not supported in Backtesting. + """ + return Trade.get_trades(Trade.open_order_id.isnot(None)).all() + + @staticmethod + def get_open_trades_without_assigned_fees(): + """ + Returns all open trades which don't have open fees set correctly + NOTE: Not supported in Backtesting. + """ + return Trade.get_trades([Trade.fee_open_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(True), + ]).all() + + @staticmethod + def get_sold_trades_without_assigned_fees(): + """ + Returns all closed trades which don't have fees set correctly + NOTE: Not supported in Backtesting. + """ + return Trade.get_trades([Trade.fee_close_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(False), + ]).all() + + @staticmethod + def total_open_trades_stakes() -> float: + """ + Calculates total invested amount in open trades + in stake currency + """ + if Trade.use_db: + total_open_stake_amount = Trade.query.with_entities( + func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar() + else: + total_open_stake_amount = sum( + t.stake_amount for t in LocalTrade.get_trades_proxy(is_open=True)) + return total_open_stake_amount or 0 + + @staticmethod + def get_overall_performance() -> List[Dict[str, Any]]: + """ + Returns List of dicts containing all Trades, including profit and trade count + NOTE: Not supported in Backtesting. + """ + pair_rates = Trade.query.with_entities( + Trade.pair, + func.sum(Trade.close_profit).label('profit_sum'), + func.sum(Trade.close_profit_abs).label('profit_sum_abs'), + func.count(Trade.pair).label('count') + ).filter(Trade.is_open.is_(False))\ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum_abs')) \ + .all() + return [ + { + 'pair': pair, + 'profit': profit, + 'profit_abs': profit_abs, + 'count': count + } + for pair, profit, profit_abs, count in pair_rates + ] + + @staticmethod + def get_best_pair(): + """ + Get best pair with closed trade. + NOTE: Not supported in Backtesting. + :returns: Tuple containing (pair, profit_sum) + """ + best_pair = Trade.query.with_entities( + Trade.pair, func.sum(Trade.close_profit).label('profit_sum') + ).filter(Trade.is_open.is_(False)) \ + .group_by(Trade.pair) \ + .order_by(desc('profit_sum')).first() + return best_pair + + class PairLock(_DECL_BASE): """ Pair Locks database model. @@ -731,8 +863,8 @@ class PairLock(_DECL_BASE): id = Column(Integer, primary_key=True) - pair = Column(String, nullable=False, index=True) - reason = Column(String, nullable=True) + pair = Column(String(25), nullable=False, index=True) + reason = Column(String(255), nullable=True) # Time the pair was locked (start time) lock_time = Column(DateTime, nullable=False) # Time until the pair is locked (end time) @@ -765,6 +897,7 @@ class PairLock(_DECL_BASE): def to_json(self) -> Dict[str, Any]: return { + 'id': self.id, 'pair': self.pair, 'lock_time': self.lock_time.strftime(DATETIME_PRINT_FORMAT), 'lock_timestamp': int(self.lock_time.replace(tzinfo=timezone.utc).timestamp() * 1000), diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index 8644146d8..af904f693 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -48,8 +48,8 @@ class PairLocks(): active=True ) if PairLocks.use_db: - PairLock.session.add(lock) - PairLock.session.flush() + PairLock.query.session.add(lock) + PairLock.query.session.commit() else: PairLocks.locks.append(lock) @@ -99,7 +99,7 @@ class PairLocks(): for lock in locks: lock.active = False if PairLocks.use_db: - PairLock.session.flush() + PairLock.query.session.commit() @staticmethod def is_global_lock(now: Optional[datetime] = None) -> bool: @@ -123,3 +123,11 @@ class PairLocks(): now = datetime.now(timezone.utc) return len(PairLocks.get_pair_locks(pair, now)) > 0 or PairLocks.is_global_lock(now) + + @staticmethod + def get_all_locks() -> List[PairLock]: + + if PairLocks.use_db: + return PairLock.query.all() + else: + return PairLocks.locks diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 4325e537e..c1b1232c2 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -47,7 +47,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0): data = load_data( datadir=config.get('datadir'), pairs=pairs, - timeframe=config.get('timeframe', '5m'), + timeframe=config['timeframe'], timerange=timerange, startup_candles=startup_candles, data_format=config.get('dataformat_ohlcv', 'json'), @@ -56,7 +56,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0): if startup_candles and data: min_date, max_date = get_timerange(data) logger.info(f"Loading data from {min_date} to {max_date}") - timerange.adjust_start_if_necessary(timeframe_to_seconds(config.get('timeframe', '5m')), + timerange.adjust_start_if_necessary(timeframe_to_seconds(config['timeframe']), startup_candles, min_date) no_trades = False @@ -77,7 +77,8 @@ def init_plotscript(config, markets: List, startup_candles: int = 0): ) except ValueError as e: raise OperationalException(e) from e - trades = trim_dataframe(trades, timerange, 'open_date') + if not trades.empty: + trades = trim_dataframe(trades, timerange, 'open_date') return {"ohlcv": data, "trades": trades, @@ -95,20 +96,34 @@ def add_indicators(fig, row, indicators: Dict[str, Dict], data: pd.DataFrame) -> Dict key must correspond to dataframe column. :param data: candlestick DataFrame """ + plot_kinds = { + 'scatter': go.Scatter, + 'bar': go.Bar, + } for indicator, conf in indicators.items(): logger.debug(f"indicator {indicator} with config {conf}") if indicator in data: kwargs = {'x': data['date'], 'y': data[indicator].values, - 'mode': 'lines', 'name': indicator } - if 'color' in conf: - kwargs.update({'line': {'color': conf['color']}}) - scatter = go.Scatter( - **kwargs - ) - fig.add_trace(scatter, row, 1) + + plot_type = conf.get('type', 'scatter') + color = conf.get('color') + if plot_type == 'bar': + kwargs.update({'marker_color': color or 'DarkSlateGrey', + 'marker_line_color': color or 'DarkSlateGrey'}) + else: + if color: + kwargs.update({'line': {'color': color}}) + kwargs['mode'] = 'lines' + if plot_type != 'scatter': + logger.warning(f'Indicator {indicator} has unknown plot trace kind {plot_type}' + f', assuming "scatter".') + + kwargs.update(conf.get('plotly', {})) + trace = plot_kinds[plot_type](**kwargs) + fig.add_trace(trace, row, 1) else: logger.info( 'Indicator "%s" ignored. Reason: This indicator is not found ' @@ -145,7 +160,7 @@ def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame, Add scatter points indicating max drawdown """ try: - max_drawdown, highdate, lowdate = calculate_max_drawdown(trades) + max_drawdown, highdate, lowdate, _, _ = calculate_max_drawdown(trades) drawdown = go.Scatter( x=[highdate, lowdate], @@ -441,7 +456,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame], - trades: pd.DataFrame, timeframe: str) -> go.Figure: + trades: pd.DataFrame, timeframe: str, stake_currency: str) -> go.Figure: # Combine close-values for all pairs, rename columns to "pair" df_comb = combine_dataframes_with_mean(data, "close") @@ -466,8 +481,8 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame], subplot_titles=["AVG Close Price", "Combined Profit", "Profit per pair"]) fig['layout'].update(title="Freqtrade Profit plot") fig['layout']['yaxis1'].update(title='Price') - fig['layout']['yaxis2'].update(title='Profit') - fig['layout']['yaxis3'].update(title='Profit') + fig['layout']['yaxis2'].update(title=f'Profit {stake_currency}') + fig['layout']['yaxis3'].update(title=f'Profit {stake_currency}') fig['layout']['xaxis']['rangeslider'].update(visible=False) fig.add_trace(avgclose, 1, 1) @@ -540,8 +555,11 @@ def load_and_plot_trades(config: Dict[str, Any]): df_analyzed = strategy.analyze_ticker(data, {'pair': pair}) df_analyzed = trim_dataframe(df_analyzed, timerange) - trades_pair = trades.loc[trades['pair'] == pair] - trades_pair = extract_trades_of_period(df_analyzed, trades_pair) + if not trades.empty: + trades_pair = trades.loc[trades['pair'] == pair] + trades_pair = extract_trades_of_period(df_analyzed, trades_pair) + else: + trades_pair = trades fig = generate_candlestick_graph( pair=pair, @@ -565,6 +583,9 @@ def plot_profit(config: Dict[str, Any]) -> None: But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ + if 'timeframe' not in config: + raise OperationalException('Timeframe must be set in either config or via --timeframe.') + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) plot_elements = init_plotscript(config, list(exchange.markets)) trades = plot_elements['trades'] @@ -581,6 +602,8 @@ def plot_profit(config: Dict[str, Any]) -> None: # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'], - trades, config.get('timeframe', '5m')) + trades, config['timeframe'], + config.get('stake_currency', '')) store_plot_file(fig, filename='freqtrade-profit-plot.html', - directory=config['user_data_dir'] / 'plot', auto_open=True) + directory=config['user_data_dir'] / 'plot', + auto_open=config.get('plot_auto_open', False)) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 8c3a5d22f..8f623b062 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -30,10 +30,10 @@ class AgeFilter(IPairList): if self._min_days_listed < 1: raise OperationalException("AgeFilter requires min_days_listed to be >= 1") - if self._min_days_listed > exchange.ohlcv_candle_limit: + if self._min_days_listed > exchange.ohlcv_candle_limit('1d'): raise OperationalException("AgeFilter requires min_days_listed to not exceed " "exchange max request size " - f"({exchange.ohlcv_candle_limit})") + f"({exchange.ohlcv_candle_limit('1d')})") @property def needstickers(self) -> bool: @@ -71,14 +71,14 @@ class AgeFilter(IPairList): daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) - logger.info(f"Validated {len(pairlist)} pairs.") + self.log_once(f"Validated {len(pairlist)} pairs.", logger.info) return pairlist def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: """ Validate age for the ticker :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.load_markets() + :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache @@ -86,7 +86,7 @@ class AgeFilter(IPairList): return True if daily_candles is not None: - if len(daily_candles) > self._min_days_listed: + if len(daily_candles) >= self._min_days_listed: # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000 diff --git a/freqtrade/plugins/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py index 184feff9e..74348b1a7 100644 --- a/freqtrade/plugins/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -7,7 +7,7 @@ from copy import deepcopy from typing import Any, Dict, List from freqtrade.exceptions import OperationalException -from freqtrade.exchange import market_is_active +from freqtrade.exchange import Exchange, market_is_active from freqtrade.mixins import LoggingMixin @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) class IPairList(LoggingMixin, ABC): - def __init__(self, exchange, pairlistmanager, + def __init__(self, exchange: Exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: """ @@ -28,7 +28,7 @@ class IPairList(LoggingMixin, ABC): """ self._enabled = True - self._exchange = exchange + self._exchange: Exchange = exchange self._pairlistmanager = pairlistmanager self._config = config self._pairlistconfig = pairlistconfig @@ -68,12 +68,12 @@ class IPairList(LoggingMixin, ABC): filter_pairlist() method. :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.load_markets() + :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ raise NotImplementedError() - def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + def gen_pairlist(self, tickers: Dict) -> List[str]: """ Generate the pairlist. @@ -84,8 +84,7 @@ class IPairList(LoggingMixin, ABC): it will raise the exception if a Pairlist Handler is used at the first position in the chain. - :param cached_pairlist: Previously generated pairlist (cached) - :param tickers: Tickers (from exchange.get_tickers()). + :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: List of pairs """ raise OperationalException("This Pairlist Handler should not be used " diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py index 7d91bb77c..bf474cb21 100644 --- a/freqtrade/plugins/pairlist/PerformanceFilter.py +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -2,7 +2,7 @@ Performance pair list filter """ import logging -from typing import Any, Dict, List +from typing import Dict, List import pandas as pd @@ -15,11 +15,6 @@ logger = logging.getLogger(__name__) class PerformanceFilter(IPairList): - def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], - pairlist_pos: int) -> None: - super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - @property def needstickers(self) -> bool: """ @@ -44,7 +39,12 @@ class PerformanceFilter(IPairList): :return: new allowlist """ # Get the trading performance for pairs from database - performance = pd.DataFrame(Trade.get_overall_performance()) + try: + performance = pd.DataFrame(Trade.get_overall_performance()) + except AttributeError: + # Performancefilter does not work in backtesting. + self.log_once("PerformanceFilter is not available in this mode.", logger.warning) + return pairlist # Skip performance-based sorting if no performance data is available if len(performance) == 0: diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index 519337f29..a3c262e8c 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -48,7 +48,7 @@ class PrecisionFilter(IPairList): Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very low value pairs. :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.load_markets() + :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ stop_price = ticker['ask'] * self._stoploss diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py index 6558f196f..5b5afb557 100644 --- a/freqtrade/plugins/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -27,9 +27,13 @@ class PriceFilter(IPairList): self._max_price = pairlistconfig.get('max_price', 0) if self._max_price < 0: raise OperationalException("PriceFilter requires max_price to be >= 0") + self._max_value = pairlistconfig.get('max_value', 0) + if self._max_value < 0: + raise OperationalException("PriceFilter requires max_value to be >= 0") self._enabled = ((self._low_price_ratio > 0) or (self._min_price > 0) or - (self._max_price > 0)) + (self._max_price > 0) or + (self._max_value > 0)) @property def needstickers(self) -> bool: @@ -51,6 +55,8 @@ class PriceFilter(IPairList): active_price_filters.append(f"below {self._min_price:.8f}") if self._max_price != 0: active_price_filters.append(f"above {self._max_price:.8f}") + if self._max_value != 0: + active_price_filters.append(f"Value above {self._max_value:.8f}") if len(active_price_filters): return f"{self.name} - Filtering pairs priced {' or '.join(active_price_filters)}." @@ -61,10 +67,10 @@ class PriceFilter(IPairList): """ Check if if one price-step (pip) is > than a certain barrier. :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.load_markets() + :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ - if ticker['last'] is None or ticker['last'] == 0: + if ticker.get('last', None) is None or ticker.get('last') == 0: self.log_once(f"Removed {pair} from whitelist, because " "ticker['last'] is empty (Usually no trade in the last 24h).", logger.info) @@ -79,6 +85,32 @@ class PriceFilter(IPairList): f"because 1 unit is {changeperc * 100:.3f}%", logger.info) return False + # Perform low_amount check + if self._max_value != 0: + price = ticker['last'] + market = self._exchange.markets[pair] + limits = market['limits'] + if ('amount' in limits and 'min' in limits['amount'] + and limits['amount']['min'] is not None): + min_amount = limits['amount']['min'] + min_precision = market['precision']['amount'] + + min_value = min_amount * price + if self._exchange.precisionMode == 4: + # tick size + next_value = (min_amount + min_precision) * price + else: + # Decimal places + min_precision = pow(0.1, min_precision) + next_value = (min_amount + min_precision) * price + diff = next_value - min_value + + if diff > self._max_value: + self.log_once(f"Removed {pair} from whitelist, " + f"because min value change of {diff} > {self._max_value}.", + logger.info) + return False + # Perform min_price check. if self._min_price != 0: if ticker['last'] < self._min_price: @@ -89,7 +121,7 @@ class PriceFilter(IPairList): # Perform max_price check. if self._max_price != 0: if ticker['last'] > self._max_price: - self.log_once(f"Removed {ticker['symbol']} from whitelist, " + self.log_once(f"Removed {pair} from whitelist, " f"because last price > {self._max_price:.8f}", logger.info) return False diff --git a/freqtrade/plugins/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py index 9fa211750..1b152774b 100644 --- a/freqtrade/plugins/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -40,7 +40,7 @@ class SpreadFilter(IPairList): """ Validate spread for the ticker :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.load_markets() + :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ if 'bid' in ticker and 'ask' in ticker and ticker['ask']: diff --git a/freqtrade/plugins/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py index c5ced48c9..d8623e13d 100644 --- a/freqtrade/plugins/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -42,11 +42,10 @@ class StaticPairList(IPairList): """ return f"{self.name}" - def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + def gen_pairlist(self, tickers: Dict) -> List[str]: """ Generate the pairlist - :param cached_pairlist: Previously generated pairlist (cached) - :param tickers: Tickers (from exchange.get_tickers()). + :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: List of pairs """ if self._allow_inactive: diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py new file mode 100644 index 000000000..bc617a1db --- /dev/null +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -0,0 +1,121 @@ +""" +Volatility pairlist filter +""" +import logging +import sys +from copy import deepcopy +from typing import Any, Dict, List, Optional + +import arrow +import numpy as np +from cachetools.ttl import TTLCache +from pandas import DataFrame + +from freqtrade.exceptions import OperationalException +from freqtrade.misc import plural +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class VolatilityFilter(IPairList): + ''' + Filters pairs by volatility + ''' + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._days = pairlistconfig.get('lookback_days', 10) + self._min_volatility = pairlistconfig.get('min_volatility', 0) + self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize) + self._refresh_period = pairlistconfig.get('refresh_period', 1440) + + self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) + + if self._days < 1: + raise OperationalException("VolatilityFilter requires lookback_days to be >= 1") + if self._days > exchange.ohlcv_candle_limit('1d'): + raise OperationalException("VolatilityFilter requires lookback_days to not " + "exceed exchange max request size " + f"({exchange.ohlcv_candle_limit('1d')})") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Filtering pairs with volatility range " + f"{self._min_volatility}-{self._max_volatility} " + f" the last {self._days} {plural(self._days, 'day')}.") + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Validate trading range + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] + + since_ms = int(arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .float_timestamp) * 1000 + # Get all candles + candles = {} + if needed_pairs: + candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, + cache=False) + + if self._enabled: + for p in deepcopy(pairlist): + daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None + if not self._validate_pair_loc(p, daily_candles): + pairlist.remove(p) + return pairlist + + def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: + """ + Validate trading range + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.fetch_tickers() + :return: True if the pair can stay, false if it should be removed + """ + # Check symbol in cache + cached_res = self._pair_cache.get(pair, None) + if cached_res is not None: + return cached_res + + result = False + if daily_candles is not None and not daily_candles.empty: + returns = (np.log(daily_candles.close / daily_candles.close.shift(-1))) + returns.fillna(0, inplace=True) + + volatility_series = returns.rolling(window=self._days).std()*np.sqrt(self._days) + volatility_avg = volatility_series.mean() + + if self._min_volatility <= volatility_avg <= self._max_volatility: + result = True + else: + self.log_once(f"Removed {pair} from whitelist, because volatility " + f"over {self._days} {plural(self._days, 'day')} " + f"is: {volatility_avg:.3f} " + f"which is not in the configured range of " + f"{self._min_volatility}-{self._max_volatility}.", + logger.info) + result = False + self._pair_cache[pair] = result + + return result diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index dd8fc64fd..8eff137b0 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -4,9 +4,10 @@ Volume PairList provider Provides dynamic pair list based on trade volumes """ import logging -from datetime import datetime from typing import Any, Dict, List +from cachetools.ttl import TTLCache + from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList @@ -33,7 +34,8 @@ class VolumePairList(IPairList): self._number_pairs = self._pairlistconfig['number_assets'] self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') self._min_value = self._pairlistconfig.get('min_value', 0) - self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) + self._refresh_period = self._pairlistconfig.get('refresh_period', 1800) + self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) if not self._exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -63,17 +65,19 @@ class VolumePairList(IPairList): """ return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs." - def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]: + def gen_pairlist(self, tickers: Dict) -> List[str]: """ Generate the pairlist - :param cached_pairlist: Previously generated pairlist (cached) - :param tickers: Tickers (from exchange.get_tickers()). + :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: List of pairs """ # Generate dynamic whitelist # Must always run if this pairlist is not the first in the list. - if self._last_refresh + self.refresh_period < datetime.now().timestamp(): - self._last_refresh = int(datetime.now().timestamp()) + pairlist = self._pair_cache.get('pairlist') + if pairlist: + # Item found - no refresh necessary + return pairlist + else: # Use fresh pairlist # Check if pair quote currency equals to the stake currency. @@ -82,9 +86,9 @@ class VolumePairList(IPairList): if (self._exchange.get_pair_quote_currency(k) == self._stake_currency and v[self._sort_key] is not None)] pairlist = [s['symbol'] for s in filtered_tickers] - else: - # Use the cached pairlist if it's not time yet to refresh - pairlist = cached_pairlist + + pairlist = self.filter_pairlist(pairlist, tickers) + self._pair_cache['pairlist'] = pairlist return pairlist diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index f2e84930b..8be61166b 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -28,14 +28,14 @@ class RangeStabilityFilter(IPairList): self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01) self._refresh_period = pairlistconfig.get('refresh_period', 1440) - self._pair_cache: TTLCache = TTLCache(maxsize=100, ttl=self._refresh_period) + self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) if self._days < 1: raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1") - if self._days > exchange.ohlcv_candle_limit: + if self._days > exchange.ohlcv_candle_limit('1d'): raise OperationalException("RangeStabilityFilter requires lookback_days to not " "exceed exchange max request size " - f"({exchange.ohlcv_candle_limit})") + f"({exchange.ohlcv_candle_limit('1d')})") @property def needstickers(self) -> bool: @@ -83,12 +83,13 @@ class RangeStabilityFilter(IPairList): """ Validate trading range :param pair: Pair that's currently validated - :param ticker: ticker dict as returned from ccxt.load_markets() + :param ticker: ticker dict as returned from ccxt.fetch_tickers() :return: True if the pair can stay, false if it should be removed """ # Check symbol in cache - if pair in self._pair_cache: - return self._pair_cache[pair] + cached_res = self._pair_cache.get(pair, None) + if cached_res is not None: + return cached_res result = False if daily_candles is not None and not daily_candles.empty: diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 4e4135981..d1cdd2c5b 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -3,7 +3,7 @@ PairList manager class """ import logging from copy import deepcopy -from typing import Any, Dict, List +from typing import Dict, List from cachetools import TTLCache, cached @@ -79,11 +79,8 @@ class PairListManager(): if self._tickers_needed: tickers = self._get_cached_tickers() - # Adjust whitelist if filters are using tickers - pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers) - # Generate the pairlist with first Pairlist Handler in the chain - pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers) + pairlist = self._pairlist_handlers[0].gen_pairlist(tickers) # Process all Pairlist Handlers in the chain for pairlist_handler in self._pairlist_handlers: @@ -95,19 +92,6 @@ class PairListManager(): self._whitelist = pairlist - def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]: - """ - Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove - pairs that do not have ticker available - """ - if self._tickers_needed: - # Copy list since we're modifying this list - for p in deepcopy(pairlist): - if p not in tickers: - pairlist.remove(p) - - return pairlist - def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]: """ Verify and remove items from pairlist - returning a filtered pairlist. diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py index 2d7d7b4c7..a2d8eca34 100644 --- a/freqtrade/plugins/protections/cooldown_period.py +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -1,7 +1,6 @@ import logging from datetime import datetime, timedelta -from typing import Any, Dict from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn @@ -15,9 +14,6 @@ class CooldownPeriod(IProtection): has_global_stop: bool = False has_local_stop: bool = True - def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: - super().__init__(config, protection_config) - def _reason(self) -> str: """ LockReason to use @@ -44,7 +40,8 @@ class CooldownPeriod(IProtection): trades = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) if trades: # Get latest trade - trade = sorted(trades, key=lambda t: t.close_date)[-1] + # Ignore type error as we know we only get closed trades. + trade = sorted(trades, key=lambda t: t.close_date)[-1] # type: ignore self.log_once(f"Cooldown for {pair} for {self.stop_duration_str}.", logger.info) until = self.calculate_lock_end([trade], self._stop_duration) diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py index 684bf6cd3..d034beefc 100644 --- a/freqtrade/plugins/protections/iprotection.py +++ b/freqtrade/plugins/protections/iprotection.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional, Tuple from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import plural from freqtrade.mixins import LoggingMixin -from freqtrade.persistence import Trade +from freqtrade.persistence import LocalTrade logger = logging.getLogger(__name__) @@ -93,11 +93,11 @@ class IProtection(LoggingMixin, ABC): """ @staticmethod - def calculate_lock_end(trades: List[Trade], stop_minutes: int) -> datetime: + def calculate_lock_end(trades: List[LocalTrade], stop_minutes: int) -> datetime: """ Get lock end time """ - max_date: datetime = max([trade.close_date for trade in trades]) + max_date: datetime = max([trade.close_date for trade in trades if trade.close_date]) # comming from Database, tzinfo is not set. if max_date.tzinfo is None: max_date = max_date.replace(tzinfo=timezone.utc) diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py index 9d5ed35b4..7822ce73c 100644 --- a/freqtrade/plugins/protections/low_profit_pairs.py +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -53,7 +53,7 @@ class LowProfitPairs(IProtection): # Not enough trades in the relevant period return False, None, None - profit = sum(trade.close_profit for trade in trades) + profit = sum(trade.close_profit for trade in trades if trade.close_profit) if profit < self._required_profit: self.log_once( f"Trading for {pair} stopped due to {profit:.2f} < {self._required_profit} " diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index d54e6699b..67e204039 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -55,13 +55,13 @@ class MaxDrawdown(IProtection): # Drawdown is always positive try: - drawdown, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') + drawdown, _, _, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') except ValueError: return False, None, None if drawdown > self._max_allowed_drawdown: self.log_once( - f"Trading stopped due to Max Drawdown {drawdown:.2f} < {self._max_allowed_drawdown}" + f"Trading stopped due to Max Drawdown {drawdown:.2f} > {self._max_allowed_drawdown}" f" within {self.lookback_period_str}.", logger.info) until = self.calculate_lock_end(trades, self._stop_duration) diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 92fae54cb..45d393411 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -3,9 +3,9 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict +from freqtrade.enums import SellType from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn -from freqtrade.strategy.interface import SellType logger = logging.getLogger(__name__) @@ -56,15 +56,15 @@ class StoplossGuard(IProtection): trades = [trade for trade in trades1 if (str(trade.sell_reason) in ( SellType.TRAILING_STOP_LOSS.value, SellType.STOP_LOSS.value, SellType.STOPLOSS_ON_EXCHANGE.value) - and trade.close_profit < 0)] + and trade.close_profit and trade.close_profit < 0)] - if len(trades) > self._trade_limit: - self.log_once(f"Trading stopped due to {self._trade_limit} " - f"stoplosses within {self._lookback_period} minutes.", logger.info) - until = self.calculate_lock_end(trades, self._stop_duration) - return True, until, self._reason() + if len(trades) < self._trade_limit: + return False, None, None - return False, None, None + self.log_once(f"Trading stopped due to {self._trade_limit} " + f"stoplosses within {self._lookback_period} minutes.", logger.info) + until = self.calculate_lock_end(trades, self._stop_duration) + return True, until, self._reason() def global_stop(self, date_now: datetime) -> ProtectionReturn: """ diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 37cfd70e6..b51795e9e 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -61,7 +61,7 @@ class IResolver: module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints - except (ModuleNotFoundError, SyntaxError, ImportError) as err: + except (ModuleNotFoundError, SyntaxError, ImportError, NameError) as err: # Catch errors in case a specific module is not installed logger.warning(f"Could not import {module_path} due to '{err}'") if enum_failed: diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index b1b66e3ae..6484f900b 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -6,7 +6,6 @@ This module load custom strategies import logging import tempfile from base64 import urlsafe_b64decode -from collections import OrderedDict from inspect import getfullargspec from pathlib import Path from typing import Any, Dict, Optional @@ -139,7 +138,7 @@ class StrategyResolver(IResolver): # Sort and apply type conversions if hasattr(strategy, 'minimal_roi'): - strategy.minimal_roi = OrderedDict(sorted( + strategy.minimal_roi = dict(sorted( {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), key=lambda t: t[0])) if hasattr(strategy, 'stoploss'): @@ -196,9 +195,9 @@ class StrategyResolver(IResolver): strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args) - if any([x == 2 for x in [strategy._populate_fun_len, - strategy._buy_fun_len, - strategy._sell_fun_len]]): + if any(x == 2 for x in [strategy._populate_fun_len, + strategy._buy_fun_len, + strategy._sell_fun_len]): strategy.INTERFACE_VERSION = 1 return strategy diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index 0a0130ca7..957565e2c 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -1,3 +1,3 @@ # flake8: noqa: F401 -from .rpc import RPC, RPCException, RPCHandler, RPCMessageType +from .rpc import RPC, RPCException, RPCHandler from .rpc_manager import RPCManager diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 4faefb5fc..4d06d3ecf 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -1,5 +1,5 @@ from datetime import date, datetime -from typing import Any, Dict, List, Optional, TypeVar, Union +from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel @@ -57,19 +57,18 @@ class Count(BaseModel): class PerformanceEntry(BaseModel): pair: str profit: float + profit_abs: float count: int class Profit(BaseModel): profit_closed_coin: float - profit_closed_percent: float profit_closed_percent_mean: float profit_closed_ratio_mean: float profit_closed_percent_sum: float profit_closed_ratio_sum: float profit_closed_fiat: float profit_all_coin: float - profit_all_percent: float profit_all_percent_mean: float profit_all_ratio_mean: float profit_all_percent_sum: float @@ -113,7 +112,7 @@ class Daily(BaseModel): class ShowConfig(BaseModel): - dry_run: str + dry_run: bool stake_currency: str stake_amount: Union[float, str] max_open_trades: int @@ -153,13 +152,11 @@ class TradeSchema(BaseModel): fee_close: Optional[float] fee_close_cost: Optional[float] fee_close_currency: Optional[str] - open_date_hum: str open_date: str open_timestamp: int open_rate: float open_rate_requested: Optional[float] open_trade_value: float - close_date_hum: Optional[str] close_date: Optional[str] close_timestamp: Optional[int] close_rate: Optional[float] @@ -170,6 +167,7 @@ class TradeSchema(BaseModel): profit_ratio: Optional[float] profit_pct: Optional[float] profit_abs: Optional[float] + profit_fiat: Optional[float] sell_reason: Optional[str] sell_order_status: Optional[str] stop_loss_abs: Optional[float] @@ -192,7 +190,6 @@ class OpenTradeSchema(TradeSchema): stoploss_current_dist_ratio: Optional[float] stoploss_entry_dist: Optional[float] stoploss_entry_dist_ratio: Optional[float] - base_currency: str current_profit: float current_profit_abs: float current_profit_pct: float @@ -203,12 +200,15 @@ class OpenTradeSchema(TradeSchema): class TradeResponse(BaseModel): trades: List[TradeSchema] trades_count: int + total_trades: int -ForceBuyResponse = TypeVar('ForceBuyResponse', TradeSchema, StatusMsg) +class ForceBuyResponse(BaseModel): + __root__: Union[TradeSchema, StatusMsg] class LockModel(BaseModel): + id: int active: bool lock_end_time: str lock_end_timestamp: int @@ -223,6 +223,11 @@ class Locks(BaseModel): locks: List[LockModel] +class DeleteLockRequest(BaseModel): + pair: Optional[str] + lockid: Optional[int] + + class Logs(BaseModel): log_count: int logs: List[List] @@ -264,10 +269,11 @@ class DeleteTrade(BaseModel): class PlotConfig_(BaseModel): main_plot: Dict[str, Any] - subplots: Optional[Dict[str, Any]] + subplots: Dict[str, Any] -PlotConfig = TypeVar('PlotConfig', PlotConfig_, Dict) +class PlotConfig(BaseModel): + __root__: Union[PlotConfig_, Dict] class StrategyListResponse(BaseModel): diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 3588f2196..e907b92f0 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -11,13 +11,13 @@ from freqtrade.data.history import get_datahandler from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload, - BlacklistResponse, Count, Daily, DeleteTrade, - ForceBuyPayload, ForceBuyResponse, - ForceSellPayload, Locks, Logs, OpenTradeSchema, - PairHistory, PerformanceEntry, Ping, PlotConfig, - Profit, ResultMsg, ShowConfig, Stats, StatusMsg, - StrategyListResponse, StrategyResponse, - TradeResponse, Version, WhitelistResponse) + BlacklistResponse, Count, Daily, + DeleteLockRequest, DeleteTrade, ForceBuyPayload, + ForceBuyResponse, ForceSellPayload, Locks, Logs, + OpenTradeSchema, PairHistory, PerformanceEntry, + Ping, PlotConfig, Profit, ResultMsg, ShowConfig, + Stats, StatusMsg, StrategyListResponse, + StrategyResponse, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -82,9 +82,19 @@ def status(rpc: RPC = Depends(get_rpc)): return [] -@router.get('/trades', response_model=TradeResponse, tags=['info', 'trading']) -def trades(limit: int = 0, rpc: RPC = Depends(get_rpc)): - return rpc._rpc_trade_history(limit) +# Using the responsemodel here will cause a ~100% increase in response time (from 1s to 2s) +# on big databases. Correct response model: response_model=TradeResponse, +@router.get('/trades', tags=['info', 'trading']) +def trades(limit: int = 500, offset: int = 0, rpc: RPC = Depends(get_rpc)): + return rpc._rpc_trade_history(limit, offset=offset, order_by_id=True) + + +@router.get('/trade/{tradeid}', response_model=OpenTradeSchema, tags=['info', 'trading']) +def trade(tradeid: int = 0, rpc: RPC = Depends(get_rpc)): + try: + return rpc._rpc_trade_status([tradeid])[0] + except (RPCException, KeyError): + raise HTTPException(status_code=404, detail='Trade not found.') @router.delete('/trades/{tradeid}', response_model=DeleteTrade, tags=['info', 'trading']) @@ -111,9 +121,9 @@ def forcebuy(payload: ForceBuyPayload, rpc: RPC = Depends(get_rpc)): trade = rpc._rpc_forcebuy(payload.pair, payload.price) if trade: - return trade.to_json() + return ForceBuyResponse.parse_obj(trade.to_json()) else: - return {"status": f"Error buying pair {payload.pair}."} + return ForceBuyResponse.parse_obj({"status": f"Error buying pair {payload.pair}."}) @router.post('/forcesell', response_model=ResultMsg, tags=['trading']) @@ -136,11 +146,21 @@ def whitelist(rpc: RPC = Depends(get_rpc)): return rpc._rpc_whitelist() -@router.get('/locks', response_model=Locks, tags=['info']) +@router.get('/locks', response_model=Locks, tags=['info', 'locks']) def locks(rpc: RPC = Depends(get_rpc)): return rpc._rpc_locks() +@router.delete('/locks/{lockid}', response_model=Locks, tags=['info', 'locks']) +def delete_lock(lockid: int, rpc: RPC = Depends(get_rpc)): + return rpc._rpc_delete_lock(lockid=lockid) + + +@router.post('/locks/delete', response_model=Locks, tags=['info', 'locks']) +def delete_lock_pair(payload: DeleteLockRequest, rpc: RPC = Depends(get_rpc)): + return rpc._rpc_delete_lock(lockid=payload.lockid, pair=payload.pair) + + @router.get('/logs', response_model=Logs, tags=['info']) def logs(limit: Optional[int] = None, rpc: RPC = Depends(get_rpc)): return rpc._rpc_get_logs(limit) @@ -183,7 +203,7 @@ def pair_history(pair: str, timeframe: str, timerange: str, strategy: str, @router.get('/plot_config', response_model=PlotConfig, tags=['candle data']) def plot_config(rpc: RPC = Depends(get_rpc)): - return rpc._rpc_plot_config() + return PlotConfig.parse_obj(rpc._rpc_plot_config()) @router.get('/strategies', response_model=StrategyListResponse, tags=['strategy']) diff --git a/freqtrade/rpc/api_server/uvicorn_threaded.py b/freqtrade/rpc/api_server/uvicorn_threaded.py index 1554a8e52..2f72cb74c 100644 --- a/freqtrade/rpc/api_server/uvicorn_threaded.py +++ b/freqtrade/rpc/api_server/uvicorn_threaded.py @@ -8,12 +8,33 @@ import uvicorn class UvicornServer(uvicorn.Server): """ Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742 + + Removed install_signal_handlers() override based on changes from this commit: + https://github.com/encode/uvicorn/commit/ce2ef45a9109df8eae038c0ec323eb63d644cbc6 + + Cannot rely on asyncio.get_event_loop() to create new event loop because of this check: + https://github.com/python/cpython/blob/4d7f11e05731f67fd2c07ec2972c6cb9861d52be/Lib/asyncio/events.py#L638 + + Fix by overriding run() and forcing creation of new event loop if uvloop is available """ - def install_signal_handlers(self): + + def run(self, sockets=None): + import asyncio + """ - In the parent implementation, this starts the thread, therefore we must patch it away here. + Parent implementation calls self.config.setup_event_loop(), + but we need to create uvloop event loop manually """ - pass + try: + import uvloop # noqa + except ImportError: # pragma: no cover + from uvicorn.loops.asyncio import asyncio_setup + asyncio_setup() + else: + asyncio.set_event_loop(uvloop.new_event_loop()) + + loop = asyncio.get_event_loop() + loop.run_until_complete(self.serve(sockets=sockets)) @contextlib.contextmanager def run_in_thread(self): diff --git a/freqtrade/rpc/api_server/web_ui.py b/freqtrade/rpc/api_server/web_ui.py index 6d7e77953..a8c737e04 100644 --- a/freqtrade/rpc/api_server/web_ui.py +++ b/freqtrade/rpc/api_server/web_ui.py @@ -10,7 +10,12 @@ router_ui = APIRouter() @router_ui.get('/favicon.ico', include_in_schema=False) async def favicon(): - return FileResponse(Path(__file__).parent / 'ui/favicon.ico') + return FileResponse(str(Path(__file__).parent / 'ui/favicon.ico')) + + +@router_ui.get('/fallback_file.html', include_in_schema=False) +async def fallback(): + return FileResponse(str(Path(__file__).parent / 'ui/fallback_file.html')) @router_ui.get('/{rest_of_path:path}', include_in_schema=False) diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index f3eaa1ebc..8a5c958e9 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -2,6 +2,7 @@ import logging from ipaddress import IPv4Address from typing import Any, Dict +import rapidjson import uvicorn from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -14,6 +15,17 @@ from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler logger = logging.getLogger(__name__) +class FTJSONResponse(JSONResponse): + media_type = "application/json" + + def render(self, content: Any) -> bytes: + """ + Use rapidjson for responses + Handles NaN and Inf / -Inf in a javascript way by default. + """ + return rapidjson.dumps(content).encode("utf-8") + + class ApiServer(RPCHandler): _rpc: RPC @@ -32,6 +44,7 @@ class ApiServer(RPCHandler): self.app = FastAPI(title="Freqtrade API", docs_url='/docs' if api_config.get('enable_openapi', False) else None, redoc_url=None, + default_response_class=FTJSONResponse, ) self.configure_app(self.app, self._config) diff --git a/freqtrade/rpc/fiat_convert.py b/freqtrade/rpc/fiat_convert.py index 4e26432d4..5ae20afa1 100644 --- a/freqtrade/rpc/fiat_convert.py +++ b/freqtrade/rpc/fiat_convert.py @@ -3,11 +3,13 @@ Module that define classes to convert Crypto-currency to FIAT e.g BTC to USD """ +import datetime import logging -import time -from typing import Dict, List +from typing import Dict +from cachetools.ttl import TTLCache from pycoingecko import CoinGeckoAPI +from requests.exceptions import RequestException from freqtrade.constants import SUPPORTED_FIAT @@ -15,51 +17,6 @@ from freqtrade.constants import SUPPORTED_FIAT logger = logging.getLogger(__name__) -class CryptoFiat: - """ - Object to describe what is the price of Crypto-currency in a FIAT - """ - # Constants - CACHE_DURATION = 6 * 60 * 60 # 6 hours - - def __init__(self, crypto_symbol: str, fiat_symbol: str, price: float) -> None: - """ - Create an object that will contains the price for a crypto-currency in fiat - :param crypto_symbol: Crypto-currency you want to convert (e.g BTC) - :param fiat_symbol: FIAT currency you want to convert to (e.g USD) - :param price: Price in FIAT - """ - - # Public attributes - self.crypto_symbol = None - self.fiat_symbol = None - self.price = 0.0 - - # Private attributes - self._expiration = 0.0 - - self.crypto_symbol = crypto_symbol.lower() - self.fiat_symbol = fiat_symbol.lower() - self.set_price(price=price) - - def set_price(self, price: float) -> None: - """ - Set the price of the Crypto-currency in FIAT and set the expiration time - :param price: Price of the current Crypto currency in the fiat - :return: None - """ - self.price = price - self._expiration = time.time() + self.CACHE_DURATION - - def is_expired(self) -> bool: - """ - Return if the current price is still valid or needs to be refreshed - :return: bool, true the price is expired and needs to be refreshed, false the price is - still valid - """ - return self._expiration - time.time() <= 0 - - class CryptoToFiatConverter: """ Main class to initiate Crypto to FIAT. @@ -70,6 +27,7 @@ class CryptoToFiatConverter: _coingekko: CoinGeckoAPI = None _cryptomap: Dict = {} + _backoff: float = 0.0 def __new__(cls): """ @@ -84,14 +42,29 @@ class CryptoToFiatConverter: return CryptoToFiatConverter.__instance def __init__(self) -> None: - self._pairs: List[CryptoFiat] = [] + # Timeout: 6h + self._pair_price: TTLCache = TTLCache(maxsize=500, ttl=6 * 60 * 60) + self._load_cryptomap() def _load_cryptomap(self) -> None: try: coinlistings = self._coingekko.get_coins_list() - # Create mapping table from synbol to coingekko_id + # Create mapping table from symbol to coingekko_id self._cryptomap = {x['symbol']: x['id'] for x in coinlistings} + except RequestException as request_exception: + if "429" in str(request_exception): + logger.warning( + "Too many requests for Coingecko API, backing off and trying again later.") + # Set backoff timestamp to 60 seconds in the future + self._backoff = datetime.datetime.now().timestamp() + 60 + return + # If the request is not a 429 error we want to raise the normal error + logger.error( + "Could not load FIAT Cryptocurrency map for the following problem: {}".format( + request_exception + ) + ) except (Exception) as exception: logger.error( f"Could not load FIAT Cryptocurrency map for the following problem: {exception}") @@ -118,49 +91,31 @@ class CryptoToFiatConverter: """ crypto_symbol = crypto_symbol.lower() fiat_symbol = fiat_symbol.lower() + inverse = False + if crypto_symbol == 'usd': + # usd corresponds to "uniswap-state-dollar" for coingecko. + # We'll therefore need to "swap" the currencies + logger.info(f"reversing Rates {crypto_symbol}, {fiat_symbol}") + crypto_symbol = fiat_symbol + fiat_symbol = 'usd' + inverse = True + + symbol = f"{crypto_symbol}/{fiat_symbol}" # Check if the fiat convertion you want is supported if not self._is_supported_fiat(fiat=fiat_symbol): raise ValueError(f'The fiat {fiat_symbol} is not supported.') - # Get the pair that interest us and return the price in fiat - for pair in self._pairs: - if pair.crypto_symbol == crypto_symbol and pair.fiat_symbol == fiat_symbol: - # If the price is expired we refresh it, avoid to call the API all the time - if pair.is_expired(): - pair.set_price( - price=self._find_price( - crypto_symbol=pair.crypto_symbol, - fiat_symbol=pair.fiat_symbol - ) - ) + price = self._pair_price.get(symbol, None) - # return the last price we have for this pair - return pair.price - - # The pair does not exist, so we create it and return the price - return self._add_pair( - crypto_symbol=crypto_symbol, - fiat_symbol=fiat_symbol, - price=self._find_price( + if not price: + price = self._find_price( crypto_symbol=crypto_symbol, fiat_symbol=fiat_symbol ) - ) - - def _add_pair(self, crypto_symbol: str, fiat_symbol: str, price: float) -> float: - """ - :param crypto_symbol: Crypto-currency you want to convert (e.g BTC) - :param fiat_symbol: FIAT currency you want to convert to (e.g USD) - :return: price in FIAT - """ - self._pairs.append( - CryptoFiat( - crypto_symbol=crypto_symbol, - fiat_symbol=fiat_symbol, - price=price - ) - ) + if inverse and price != 0.0: + price = 1 / price + self._pair_price[symbol] = price return price @@ -188,6 +143,15 @@ class CryptoToFiatConverter: if crypto_symbol == fiat_symbol: return 1.0 + if self._cryptomap == {}: + if self._backoff <= datetime.datetime.now().timestamp(): + self._load_cryptomap() + # return 0.0 if we still dont have data to check, no reason to proceed + if self._cryptomap == {}: + return 0.0 + else: + return 0.0 + if crypto_symbol not in self._cryptomap: # return 0 for unsupported stake currencies (fiat-convert should not break the bot) logger.warning("unsupported crypto-symbol %s - returning 0.0", crypto_symbol) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 7549c38be..2a7721af0 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -3,8 +3,7 @@ This module contains class to define a RPC communications """ import logging from abc import abstractmethod -from datetime import date, datetime, timedelta -from enum import Enum +from datetime import date, datetime, timedelta, timezone from math import isnan from typing import Any, Dict, List, Optional, Tuple, Union @@ -15,36 +14,21 @@ from pandas import DataFrame from freqtrade.configuration.timerange import TimeRange from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT from freqtrade.data.history import load_data +from freqtrade.enums import SellType, State from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.loggers import bufferHandler from freqtrade.misc import shorten_date from freqtrade.persistence import PairLocks, Trade +from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter -from freqtrade.state import State -from freqtrade.strategy.interface import SellType +from freqtrade.strategy.interface import SellCheckTuple logger = logging.getLogger(__name__) -class RPCMessageType(Enum): - STATUS_NOTIFICATION = 'status' - WARNING_NOTIFICATION = 'warning' - STARTUP_NOTIFICATION = 'startup' - BUY_NOTIFICATION = 'buy' - BUY_CANCEL_NOTIFICATION = 'buy_cancel' - SELL_NOTIFICATION = 'sell' - SELL_CANCEL_NOTIFICATION = 'sell_cancel' - - def __repr__(self): - return self.value - - def __str__(self): - return self.value - - class RPCException(Exception): """ Should be raised with a rpc-formatted message in an _rpc_* method @@ -166,12 +150,24 @@ class RPC: if trade.open_order_id: order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) # calculate profit and send message to user - try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) - except (ExchangeError, PricingError): - current_rate = NAN + if trade.is_open: + try: + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) + except (ExchangeError, PricingError): + current_rate = NAN + else: + current_rate = trade.close_rate current_profit = trade.calc_profit_ratio(current_rate) current_profit_abs = trade.calc_profit(current_rate) + current_profit_fiat: Optional[float] = None + # Calculate fiat profit + if self._fiat_converter: + current_profit_fiat = self._fiat_converter.convert_amount( + current_profit_abs, + self._freqtrade.config['stake_currency'], + self._freqtrade.config['fiat_display_currency'] + ) + # Calculate guaranteed profit (in case of trailing stop) stoploss_entry_dist = trade.calc_profit(trade.stop_loss) stoploss_entry_dist_ratio = trade.calc_profit_ratio(trade.stop_loss) @@ -190,6 +186,7 @@ class RPC: profit_ratio=current_profit, profit_pct=round(current_profit * 100, 2), profit_abs=current_profit_abs, + profit_fiat=current_profit_fiat, stoploss_current_dist=stoploss_current_dist, stoploss_current_dist_ratio=round(stoploss_current_dist_ratio, 8), @@ -204,16 +201,17 @@ class RPC: return results def _rpc_status_table(self, stake_currency: str, - fiat_display_currency: str) -> Tuple[List, List]: + fiat_display_currency: str) -> Tuple[List, List, float]: trades = Trade.get_open_trades() if not trades: raise RPCException('no active trade') else: trades_list = [] + fiat_profit_sum = NAN for trade in trades: # calculate profit and send message to user try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (PricingError, ExchangeError): current_rate = NAN trade_percent = (100 * trade.calc_profit_ratio(current_rate)) @@ -227,6 +225,8 @@ class RPC: ) if fiat_profit and not isnan(fiat_profit): profit_str += f" ({fiat_profit:.2f})" + fiat_profit_sum = fiat_profit if isnan(fiat_profit_sum) \ + else fiat_profit_sum + fiat_profit trades_list.append([ trade.id, trade.pair + ('*' if (trade.open_order_id is not None @@ -240,7 +240,7 @@ class RPC: profitcol += " (" + fiat_display_currency + ")" columns = ['ID', 'Pair', 'Since', profitcol] - return trades_list, columns + return trades_list, columns, fiat_profit_sum def _rpc_daily_profit( self, timescale: int, @@ -284,19 +284,22 @@ class RPC: 'data': data } - def _rpc_trade_history(self, limit: int) -> Dict: + def _rpc_trade_history(self, limit: int, offset: int = 0, order_by_id: bool = False) -> Dict: """ Returns the X last trades """ - if limit > 0: + order_by = Trade.id if order_by_id else Trade.close_date.desc() + if limit: trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by( - Trade.id.desc()).limit(limit) + order_by).limit(limit).offset(offset) else: - trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(Trade.id.desc()).all() + trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by( + Trade.close_date.desc()).all() output = [trade.to_json() for trade in trades] return { "trades": output, - "trades_count": len(output) + "trades_count": len(output), + "total_trades": Trade.get_trades([Trade.is_open.is_(False)]).count(), } def _rpc_stats(self) -> Dict[str, Any]: @@ -333,9 +336,10 @@ class RPC: return {'sell_reasons': sell_reasons, 'durations': durations} def _rpc_trade_statistics( - self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: + self, stake_currency: str, fiat_display_currency: str, + start_date: datetime = datetime.fromtimestamp(0)) -> Dict[str, Any]: """ Returns cumulative profit statistics """ - trades = Trade.get_trades().order_by(Trade.id).all() + trades = Trade.get_trades([Trade.open_date >= start_date]).order_by(Trade.id).all() profit_all_coin = [] profit_all_ratio = [] @@ -364,7 +368,7 @@ class RPC: else: # Get current rate try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (PricingError, ExchangeError): current_rate = NAN profit_ratio = trade.calc_profit_ratio(rate=current_rate) @@ -401,14 +405,12 @@ class RPC: num = float(len(durations) or 1) return { 'profit_closed_coin': profit_closed_coin_sum, - 'profit_closed_percent': round(profit_closed_ratio_mean * 100, 2), # DEPRECATED 'profit_closed_percent_mean': round(profit_closed_ratio_mean * 100, 2), 'profit_closed_ratio_mean': profit_closed_ratio_mean, 'profit_closed_percent_sum': round(profit_closed_ratio_sum * 100, 2), 'profit_closed_ratio_sum': profit_closed_ratio_sum, 'profit_closed_fiat': profit_closed_fiat, 'profit_all_coin': profit_all_coin_sum, - 'profit_all_percent': round(profit_all_ratio_mean * 100, 2), # DEPRECATED 'profit_all_percent_mean': round(profit_all_ratio_mean * 100, 2), 'profit_all_ratio_mean': profit_all_ratio_mean, 'profit_all_percent_sum': round(profit_all_ratio_sum * 100, 2), @@ -432,7 +434,7 @@ class RPC: output = [] total = 0.0 try: - tickers = self._freqtrade.exchange.get_tickers() + tickers = self._freqtrade.exchange.get_tickers(cached=True) except (ExchangeError): raise RPCException('Error getting current tickers.') @@ -536,8 +538,9 @@ class RPC: if not fully_canceled: # Get current rate and execute sell - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) - self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) + sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL) + self._freqtrade.execute_sell(trade, current_rate, sell_reason) # ---- EOF def _exec_forcesell ---- if self._freqtrade.state != State.RUNNING: @@ -548,7 +551,7 @@ class RPC: # Execute sell for all open orders for trade in Trade.get_open_trades(): _exec_forcesell(trade) - Trade.session.flush() + Trade.commit() self._freqtrade.wallets.update() return {'result': 'Created sell orders for all open trades.'} @@ -561,7 +564,7 @@ class RPC: raise RPCException('invalid argument') _exec_forcesell(trade) - Trade.session.flush() + Trade.commit() self._freqtrade.wallets.update() return {'result': f'Created sell order for trade {trade_id}.'} @@ -590,11 +593,11 @@ class RPC: raise RPCException(f'position for {pair} already open - id: {trade.id}') # gen stake amount - stakeamount = self._freqtrade.wallets.get_trade_stake_amount( - pair, self._freqtrade.get_free_open_trades()) + stakeamount = self._freqtrade.wallets.get_trade_stake_amount(pair) # execute buy - if self._freqtrade.execute_buy(pair, stakeamount, price): + if self._freqtrade.execute_buy(pair, stakeamount, price, forcebuy=True): + Trade.commit() trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() return trade else: @@ -663,7 +666,7 @@ class RPC: } def _rpc_locks(self) -> Dict[str, Any]: - """ Returns the current locks""" + """ Returns the current locks """ locks = PairLocks.get_pair_locks(None) return { @@ -671,6 +674,24 @@ class RPC: 'locks': [lock.to_json() for lock in locks] } + def _rpc_delete_lock(self, lockid: Optional[int] = None, + pair: Optional[str] = None) -> Dict[str, Any]: + """ Delete specific lock(s) """ + locks = [] + + if pair: + locks = PairLocks.get_pair_locks(pair) + if lockid: + locks = PairLock.query.filter(PairLock.id == lockid).all() + + for lock in locks: + lock.active = False + lock.lock_end_time = datetime.now(timezone.utc) + + PairLock.query.session.commit() + + return self._rpc_locks() + def _rpc_whitelist(self) -> Dict: """ Returns the currently active whitelist""" res = {'method': self._freqtrade.pairlists.name_list, @@ -809,5 +830,7 @@ class RPC: df_analyzed, arrow.Arrow.utcnow().datetime) def _rpc_plot_config(self) -> Dict[str, Any]: - + if (self._freqtrade.strategy.plot_config and + 'subplots' not in self._freqtrade.strategy.plot_config): + self._freqtrade.strategy.plot_config['subplots'] = {} return self._freqtrade.strategy.plot_config diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 7977d68de..18ed68041 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -4,7 +4,8 @@ This module contains class to manage RPC communications (Telegram, Slack, ...) import logging from typing import Any, Dict, List -from freqtrade.rpc import RPC, RPCHandler, RPCMessageType +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPC, RPCHandler logger = logging.getLogger(__name__) @@ -67,7 +68,7 @@ class RPCManager: def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None: if config['dry_run']: self.send_msg({ - 'type': RPCMessageType.WARNING_NOTIFICATION, + 'type': RPCMessageType.WARNING, 'status': 'Dry run is enabled. All trades are simulated.' }) stake_currency = config['stake_currency'] @@ -79,7 +80,7 @@ class RPCManager: exchange_name = config['exchange']['name'] strategy_name = config.get('strategy', '') self.send_msg({ - 'type': RPCMessageType.STARTUP_NOTIFICATION, + 'type': RPCMessageType.STARTUP, 'status': f'*Exchange:* `{exchange_name}`\n' f'*Stake per trade:* `{stake_amount} {stake_currency}`\n' f'*Minimum ROI:* `{minimal_roi}`\n' @@ -88,13 +89,13 @@ class RPCManager: f'*Strategy:* `{strategy_name}`' }) self.send_msg({ - 'type': RPCMessageType.STARTUP_NOTIFICATION, + 'type': RPCMessageType.STARTUP, 'status': f'Searching for {stake_currency} pairs to buy and sell ' f'based on {pairlist.short_desc()}' }) if len(protections.name_list) > 0: prots = '\n'.join([p for prot in protections.short_desc() for k, p in prot.items()]) self.send_msg({ - 'type': RPCMessageType.STARTUP_NOTIFICATION, + 'type': RPCMessageType.STARTUP, 'status': f'Using Protections: \n{prots}' }) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 87cd7b43d..ba9c6c0f6 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,21 +5,27 @@ This module manage Telegram communication """ import json import logging -from datetime import timedelta, datetime +import re +from datetime import date, datetime, timedelta +from html import escape from itertools import chain -from typing import Any, Callable, Dict, List, Union +from math import isnan +from typing import Any, Callable, Dict, List, Union, cast import arrow from tabulate import tabulate -from telegram import KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update, InlineKeyboardButton, InlineKeyboardMarkup -from telegram.error import NetworkError, TelegramError -from telegram.ext import CallbackContext, CommandHandler, Updater, CallbackQueryHandler +from telegram import (InlineKeyboardButton, InlineKeyboardMarkup, KeyboardButton, ParseMode, + ReplyKeyboardMarkup, Update) +from telegram.error import BadRequest, NetworkError, TelegramError +from telegram.ext import CallbackContext, CallbackQueryHandler, CommandHandler, Updater from telegram.utils.helpers import escape_markdown from freqtrade.__init__ import __version__ +from freqtrade.constants import DUST_PER_COIN +from freqtrade.enums import RPCMessageType from freqtrade.exceptions import OperationalException -from freqtrade.misc import round_coin_value -from freqtrade.rpc import RPC, RPCException, RPCHandler, RPCMessageType +from freqtrade.misc import chunks, round_coin_value +from freqtrade.rpc import RPC, RPCException, RPCHandler logger = logging.getLogger(__name__) @@ -45,7 +51,7 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: cchat_id = int(update.callback_query.message.chat.id) else: cchat_id = int(update.message.chat_id) - + chat_id = int(self._config['telegram']['chat_id']) if cchat_id != chat_id: logger.info( @@ -89,7 +95,7 @@ class Telegram(RPCHandler): Validates the keyboard configuration from telegram config section. """ - self._keyboard: List[List[Union[str, KeyboardButton]]] = [ + self._keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = [ ['/daily', '/profit', '/balance'], ['/status', '/status table', '/performance'], ['/count', '/start', '/stop', '/help'] @@ -99,23 +105,27 @@ class Telegram(RPCHandler): # TODO: DRY! - its not good to list all valid cmds here. But otherwise # this needs refacoring of the whole telegram module (same # problem in _help()). - valid_keys: List[str] = ['/start', '/stop', '/status', '/status table', - '/trades', '/profit', '/performance', '/daily', - '/stats', '/count', '/locks', '/balance', - '/stopbuy', '/reload_config', '/show_config', - '/logs', '/whitelist', '/blacklist', '/edge', - '/help', '/version'] + valid_keys: List[str] = [r'/start$', r'/stop$', r'/status$', r'/status table$', + r'/trades$', r'/performance$', r'/daily$', r'/daily \d+$', + r'/profit$', r'/profit \d+', + r'/stats$', r'/count$', r'/locks$', r'/balance$', + r'/stopbuy$', r'/reload_config$', r'/show_config$', + r'/logs$', r'/whitelist$', r'/blacklist$', r'/edge$', + r'/forcebuy$', r'/help$', r'/version$'] + # Create keys for generation + valid_keys_print = [k.replace('$', '') for k in valid_keys] # custom keyboard specified in config.json cust_keyboard = self._config['telegram'].get('keyboard', []) if cust_keyboard: + combined = "(" + ")|(".join(valid_keys) + ")" # check for valid shortcuts invalid_keys = [b for b in chain.from_iterable(cust_keyboard) - if b not in valid_keys] + if not re.match(combined, b)] if len(invalid_keys): err_msg = ('config.telegram.keyboard: Invalid commands for ' f'custom Telegram keyboard: {invalid_keys}' - f'\nvalid commands are: {valid_keys}') + f'\nvalid commands are: {valid_keys_print}') raise OperationalException(err_msg) else: self._keyboard = cust_keyboard @@ -147,6 +157,7 @@ class Telegram(RPCHandler): CommandHandler('daily', self._daily), CommandHandler('count', self._count), CommandHandler('locks', self._locks), + CommandHandler(['unlock', 'delete_locks'], self._delete_locks), CommandHandler(['reload_config', 'reload_conf'], self._reload_config), CommandHandler(['show_config', 'show_conf'], self._show_config), CommandHandler('stopbuy', self._stopbuy), @@ -163,7 +174,8 @@ class Telegram(RPCHandler): CallbackQueryHandler(self._profit, pattern='update_profit'), CallbackQueryHandler(self._balance, pattern='update_balance'), CallbackQueryHandler(self._performance, pattern='update_performance'), - CallbackQueryHandler(self._count, pattern='update_count') + CallbackQueryHandler(self._count, pattern='update_count'), + CallbackQueryHandler(self._forcebuy_inline), ] for handle in handles: self._updater.dispatcher.add_handler(handle) @@ -172,10 +184,10 @@ class Telegram(RPCHandler): self._updater.dispatcher.add_handler(handle) self._updater.start_polling( - clean=True, bootstrap_retries=-1, timeout=30, read_latency=60, + drop_pending_updates=True, ) logger.info( 'rpc.telegram is listening for following commands: %s', @@ -189,79 +201,111 @@ class Telegram(RPCHandler): """ self._updater.stop() + def _format_buy_msg(self, msg: Dict[str, Any]) -> str: + if self._rpc._fiat_converter: + msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount( + msg['stake_amount'], msg['stake_currency'], msg['fiat_currency']) + else: + msg['stake_amount_fiat'] = 0 + + message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}" + f" (#{msg['trade_id']})\n" + f"*Amount:* `{msg['amount']:.8f}`\n" + f"*Open Rate:* `{msg['limit']:.8f}`\n" + f"*Current Rate:* `{msg['current_rate']:.8f}`\n" + f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}") + + if msg.get('fiat_currency', None): + message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}" + message += ")`" + return message + + def _format_sell_msg(self, msg: Dict[str, Any]) -> str: + msg['amount'] = round(msg['amount'], 8) + msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2) + msg['duration'] = msg['close_date'].replace( + microsecond=0) - msg['open_date'].replace(microsecond=0) + msg['duration_min'] = msg['duration'].total_seconds() / 60 + + msg['emoji'] = self._get_sell_emoji(msg) + + # Check if all sell properties are available. + # This might not be the case if the message origin is triggered by /forcesell + if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency']) + and self._rpc._fiat_converter): + msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount( + msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) + msg['profit_extra'] = (' ({gain}: {profit_amount:.8f} {stake_currency}' + ' / {profit_fiat:.3f} {fiat_currency})').format(**msg) + else: + msg['profit_extra'] = '' + + message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n" + "*Profit:* `{profit_percent:.2f}%{profit_extra}`\n" + "*Sell Reason:* `{sell_reason}`\n" + "*Duration:* `{duration} ({duration_min:.1f} min)`\n" + "*Amount:* `{amount:.8f}`\n" + "*Open Rate:* `{open_rate:.8f}`\n" + "*Current Rate:* `{current_rate:.8f}`\n" + "*Close Rate:* `{limit:.8f}`").format(**msg) + + return message + def send_msg(self, msg: Dict[str, Any]) -> None: """ Send a message to telegram channel """ - noti = self._config['telegram'].get('notification_settings', {} - ).get(str(msg['type']), 'on') + default_noti = 'on' + + msg_type = msg['type'] + noti = '' + if msg_type == RPCMessageType.SELL: + sell_noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), {}) + # For backward compatibility sell still can be string + if isinstance(sell_noti, str): + noti = sell_noti + else: + noti = sell_noti.get(str(msg['sell_reason']), default_noti) + else: + noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), default_noti) + if noti == 'off': - logger.info(f"Notification '{msg['type']}' not sent.") + logger.info(f"Notification '{msg_type}' not sent.") # Notification disabled return - if msg['type'] == RPCMessageType.BUY_NOTIFICATION: - if self._rpc._fiat_converter: - msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount( - msg['stake_amount'], msg['stake_currency'], msg['fiat_currency']) - else: - msg['stake_amount_fiat'] = 0 + if msg_type == RPCMessageType.BUY: + message = self._format_buy_msg(msg) - message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}\n" - f"*Amount:* `{msg['amount']:.8f}`\n" - f"*Open Rate:* `{msg['limit']:.8f}`\n" - f"*Current Rate:* `{msg['current_rate']:.8f}`\n" - f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}") - - if msg.get('fiat_currency', None): - message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}" - message += ")`" - - elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: + elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL): + msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell' message = ("\N{WARNING SIGN} *{exchange}:* " - "Cancelling open buy Order for {pair}. Reason: {reason}.".format(**msg)) + "Cancelling open {message_side} Order for {pair} (#{trade_id}). " + "Reason: {reason}.".format(**msg)) - elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: - msg['amount'] = round(msg['amount'], 8) - msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2) - msg['duration'] = msg['close_date'].replace( - microsecond=0) - msg['open_date'].replace(microsecond=0) - msg['duration_min'] = msg['duration'].total_seconds() / 60 + elif msg_type == RPCMessageType.BUY_FILL: + message = ("\N{LARGE CIRCLE} *{exchange}:* " + "Buy order for {pair} (#{trade_id}) filled " + "for {open_rate}.".format(**msg)) + elif msg_type == RPCMessageType.SELL_FILL: + message = ("\N{LARGE CIRCLE} *{exchange}:* " + "Sell order for {pair} (#{trade_id}) filled " + "for {close_rate}.".format(**msg)) + elif msg_type == RPCMessageType.SELL: + message = self._format_sell_msg(msg) - msg['emoji'] = self._get_sell_emoji(msg) - - message = ("{emoji} *{exchange}:* Selling {pair}\n" - "*Amount:* `{amount:.8f}`\n" - "*Open Rate:* `{open_rate:.8f}`\n" - "*Current Rate:* `{current_rate:.8f}`\n" - "*Close Rate:* `{limit:.8f}`\n" - "*Sell Reason:* `{sell_reason}`\n" - "*Duration:* `{duration} ({duration_min:.1f} min)`\n" - "*Profit:* `{profit_percent:.2f}%`").format(**msg) - - # Check if all sell properties are available. - # This might not be the case if the message origin is triggered by /forcesell - if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency']) - and self._rpc._fiat_converter): - msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount( - msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) - message += (' `({gain}: {profit_amount:.8f} {stake_currency}' - ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg) - - elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION: - message = ("\N{WARNING SIGN} *{exchange}:* Cancelling Open Sell Order " - "for {pair}. Reason: {reason}").format(**msg) - - elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: + elif msg_type == RPCMessageType.STATUS: message = '*Status:* `{status}`'.format(**msg) - elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION: + elif msg_type == RPCMessageType.WARNING: message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) - elif msg['type'] == RPCMessageType.STARTUP_NOTIFICATION: + elif msg_type == RPCMessageType.STARTUP: message = '{status}'.format(**msg) else: - raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) + raise NotImplementedError('Unknown message type: {}'.format(msg_type)) self._send_msg(message, disable_notification=(noti == 'silent')) @@ -305,6 +349,7 @@ class Telegram(RPCHandler): messages = [] for r in results: + r['open_date_hum'] = arrow.get(r['open_date']).humanize() lines = [ "*Trade ID:* `{trade_id}` `(since {open_date_hum})`", "*Current Pair:* {pair}", @@ -351,15 +396,41 @@ class Telegram(RPCHandler): :return: None """ try: - statlist, head = self._rpc._rpc_status_table( - self._config['stake_currency'], self._config.get('fiat_display_currency', '')) + fiat_currency = self._config.get('fiat_display_currency', '') + statlist, head, fiat_profit_sum = self._rpc._rpc_status_table( + self._config['stake_currency'], fiat_currency) - message = tabulate(statlist, headers=head, tablefmt='simple') - if(update.callback_query): - query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=f"
{message}
", parse_mode=ParseMode.HTML, callback_path="update_status_table", reload_able=True) - else: - self._send_msg(f"
{message}
", reload_able=True, callback_path="update_status_table", parse_mode=ParseMode.HTML) + show_total = not isnan(fiat_profit_sum) and len(statlist) > 1 + max_trades_per_msg = 50 + """ + Calculate the number of messages of 50 trades per message + 0.99 is used to make sure that there are no extra (empty) messages + As an example with 50 trades, there will be int(50/50 + 0.99) = 1 message + """ + messages_count = max(int(len(statlist) / max_trades_per_msg + 0.99), 1) + for i in range(0, messages_count): + trades = statlist[i * max_trades_per_msg:(i + 1) * max_trades_per_msg] + if show_total and i == messages_count - 1: + # append total line + trades.append(["Total", "", "", f"{fiat_profit_sum:.2f} {fiat_currency}"]) + + message = tabulate(trades, + headers=head, + tablefmt='simple') + if show_total and i == messages_count - 1: + # insert separators line between Total + lines = message.split("\n") + message = "\n".join(lines[:-1] + [lines[1]] + [lines[-1]]) + if(messages_count == 1 and update.callback_query): + query = update.callback_query + self._update_msg(chat_id=query.message.chat_id, + message_id=query.message.message_id, + msg=f"
{message}
", + parse_mode=ParseMode.HTML, + callback_path="update_status_table", reload_able=True) + else: + self._send_msg(f"
{message}
", reload_able=True, + callback_path="update_status_table", parse_mode=ParseMode.HTML) except RPCException as e: self._send_msg(str(e)) @@ -399,9 +470,12 @@ class Telegram(RPCHandler): message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' if(update.callback_query): query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily", reload_able=True) + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + msg=message, parse_mode=ParseMode.HTML, + callback_path="update_daily", reload_able=True) else: - self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily", reload_able=True) + self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_daily", + reload_able=True) except RPCException as e: self._send_msg(str(e)) @@ -417,9 +491,20 @@ class Telegram(RPCHandler): stake_cur = self._config['stake_currency'] fiat_disp_cur = self._config.get('fiat_display_currency', '') + start_date = datetime.fromtimestamp(0) + timescale = None + try: + if context.args: + timescale = int(context.args[0]) + today_start = datetime.combine(date.today(), datetime.min.time()) + start_date = today_start - timedelta(days=timescale) + except (TypeError, ValueError, IndexError): + pass + stats = self._rpc._rpc_trade_statistics( stake_cur, - fiat_disp_cur) + fiat_disp_cur, + start_date) profit_closed_coin = stats['profit_closed_coin'] profit_closed_percent_mean = stats['profit_closed_percent_mean'] profit_closed_percent_sum = stats['profit_closed_percent_sum'] @@ -447,22 +532,25 @@ class Telegram(RPCHandler): else: markdown_msg = "`No closed trade` \n" - markdown_msg += (f"*ROI:* All trades\n" - f"∙ `{round_coin_value(profit_all_coin, stake_cur)} " - f"({profit_all_percent_mean:.2f}%) " - f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" - f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n" - f"*Total Trade Count:* `{trade_count}`\n" - f"*First Trade opened:* `{first_trade_date}`\n" - f"*Latest Trade opened:* `{latest_trade_date}\n`" - f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" - ) + markdown_msg += ( + f"*ROI:* All trades\n" + f"∙ `{round_coin_value(profit_all_coin, stake_cur)} " + f"({profit_all_percent_mean:.2f}%) " + f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" + f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n" + f"*Total Trade Count:* `{trade_count}`\n" + f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* " + f"`{first_trade_date}`\n" + f"*Latest Trade opened:* `{latest_trade_date}\n`" + f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" + ) if stats['closed_trade_count'] > 0: markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") if(update.callback_query): query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=markdown_msg, callback_path="update_profit", reload_able=True) + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + msg=markdown_msg, callback_path="update_profit", reload_able=True) else: self._send_msg(msg=markdown_msg, callback_path="update_profit", reload_able=True) @@ -515,6 +603,10 @@ class Telegram(RPCHandler): result = self._rpc._rpc_balance(self._config['stake_currency'], self._config.get('fiat_display_currency', '')) + balance_dust_level = self._config['telegram'].get('balance_dust_level', 0.0) + if not balance_dust_level: + balance_dust_level = DUST_PER_COIN.get(self._config['stake_currency'], 1.0) + output = '' if self._config['dry_run']: output += ( @@ -524,7 +616,7 @@ class Telegram(RPCHandler): f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" ) for curr in result['currencies']: - if curr['est_stake'] > 0.0001: + if curr['est_stake'] > balance_dust_level: curr_output = ( f"*{curr['currency']}:*\n" f"\t`Available: {curr['free']:.8f}`\n" @@ -533,7 +625,8 @@ class Telegram(RPCHandler): f"\t`Est. {curr['stake']}: " f"{round_coin_value(curr['est_stake'], curr['stake'], False)}`\n") else: - curr_output = f"*{curr['currency']}:* not showing <1$ amount \n" + curr_output = (f"*{curr['currency']}:* not showing <{balance_dust_level} " + f"{curr['stake']} amount \n") # Handle overflowing messsage length if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH: @@ -548,7 +641,8 @@ class Telegram(RPCHandler): f"{round_coin_value(result['value'], result['symbol'], False)}`\n") if(update.callback_query): query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=output, callback_path="update_balance", reload_able=True) + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + msg=output, callback_path="update_balance", reload_able=True) else: self._send_msg(msg=output, callback_path="update_balance", reload_able=True) except RPCException as e: @@ -623,6 +717,25 @@ class Telegram(RPCHandler): except RPCException as e: self._send_msg(str(e)) + def _forcebuy_action(self, pair, price=None): + try: + self._rpc._rpc_forcebuy(pair, price) + except RPCException as e: + self._send_msg(str(e)) + + def _forcebuy_inline(self, update: Update, _: CallbackContext) -> None: + if update.callback_query: + query = update.callback_query + pair = query.data + query.answer() + query.edit_message_text(text=f"Force Buying: {pair}") + self._forcebuy_action(pair) + + @staticmethod + def _layout_inline_keyboard(buttons: List[InlineKeyboardButton], + cols=3) -> List[List[InlineKeyboardButton]]: + return [buttons[i:i + cols] for i in range(0, len(buttons), cols)] + @authorized_only def _forcebuy(self, update: Update, context: CallbackContext) -> None: """ @@ -635,10 +748,13 @@ class Telegram(RPCHandler): if context.args: pair = context.args[0] price = float(context.args[1]) if len(context.args) > 1 else None - try: - self._rpc._rpc_forcebuy(pair, price) - except RPCException as e: - self._send_msg(str(e)) + self._forcebuy_action(pair, price) + else: + whitelist = self._rpc._rpc_whitelist()['whitelist'] + pairs = [InlineKeyboardButton(text=pair, callback_data=pair) for pair in whitelist] + + self._send_msg(msg="Which pair?", + keyboard=self._layout_inline_keyboard(pairs)) @authorized_only def _trades(self, update: Update, context: CallbackContext) -> None: @@ -659,13 +775,13 @@ class Telegram(RPCHandler): nrecent ) trades_tab = tabulate( - [[arrow.get(trade['open_date']).humanize(), - trade['pair'], + [[arrow.get(trade['close_date']).humanize(), + trade['pair'] + " (#" + str(trade['trade_id']) + ")", f"{(100 * trade['close_profit']):.2f}% ({trade['close_profit_abs']})"] for trade in trades['trades']], headers=[ - 'Open Date', - 'Pair', + 'Close Date', + 'Pair (ID)', f'Profit ({stake_cur})', ], tablefmt='simple') @@ -708,18 +824,30 @@ class Telegram(RPCHandler): """ try: trades = self._rpc._rpc_performance() - stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( - index=i + 1, - pair=trade['pair'], - profit=trade['profit'], - count=trade['count'] - ) for i, trade in enumerate(trades)) - message = 'Performance:\n{}'.format(stats) - if(update.callback_query): + output = "Performance:\n" + sent_messages = 0 + for i, trade in enumerate(trades): + stat_line = ( + f"{i+1}.\t {trade['pair']}\t" + f"{round_coin_value(trade['profit_abs'], self._config['stake_currency'])} " + f"({trade['profit']:.2f}%) " + f"({trade['count']})\n") + + if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH: + self._send_msg(output, parse_mode=ParseMode.HTML) + output = stat_line + sent_messages += 1 + else: + output += stat_line + + if(sent_messages == 0 and update.callback_query): query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=message, parse_mode=ParseMode.HTML, callback_path="update_performance", reload_able=True) + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + msg=output, parse_mode=ParseMode.HTML, + callback_path="update_performance", reload_able=True) else: - self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_performance", reload_able=True) + self._send_msg(msg=output, parse_mode=ParseMode.HTML, + callback_path="update_performance", reload_able=True) except RPCException as e: self._send_msg(str(e)) @@ -741,9 +869,12 @@ class Telegram(RPCHandler): logger.debug(message) if(update.callback_query): query = update.callback_query - self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, msg=message, parse_mode=ParseMode.HTML, callback_path="update_count", reload_able=True) + self._update_msg(chat_id=query.message.chat_id, message_id=query.message.message_id, + msg=message, parse_mode=ParseMode.HTML, + callback_path="update_count", reload_able=True) else: - self._send_msg(msg=message, parse_mode=ParseMode.HTML, callback_path="update_count", reload_able=True) + self._send_msg(msg=message, parse_mode=ParseMode.HTML, + callback_path="update_count", reload_able=True) except RPCException as e: self._send_msg(str(e)) @@ -753,19 +884,39 @@ class Telegram(RPCHandler): Handler for /locks. Returns the currently active locks """ - try: - locks = self._rpc._rpc_locks() + rpc_locks = self._rpc._rpc_locks() + if not rpc_locks['locks']: + self._send_msg('No active locks.', parse_mode=ParseMode.HTML) + + for locks in chunks(rpc_locks['locks'], 25): message = tabulate([[ + lock['id'], lock['pair'], lock['lock_end_time'], - lock['reason']] for lock in locks['locks']], - headers=['Pair', 'Until', 'Reason'], + lock['reason']] for lock in locks], + headers=['ID', 'Pair', 'Until', 'Reason'], tablefmt='simple') - message = "
{}
".format(message) + message = f"
{escape(message)}
" logger.debug(message) self._send_msg(message, parse_mode=ParseMode.HTML) - except RPCException as e: - self._send_msg(str(e)) + + @authorized_only + def _delete_locks(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /delete_locks. + Returns the currently active locks + """ + arg = context.args[0] if context.args and len(context.args) > 0 else None + lockid = None + pair = None + if arg: + try: + lockid = int(arg) + except ValueError: + pair = arg + + self._rpc._rpc_delete_lock(lockid=lockid, pair=pair) + self._locks(update, context) @authorized_only def _whitelist(self, update: Update, context: CallbackContext) -> None: @@ -847,9 +998,17 @@ class Telegram(RPCHandler): """ try: edge_pairs = self._rpc._rpc_edge() - edge_pairs_tab = tabulate(edge_pairs, headers='keys', tablefmt='simple') - message = f'Edge only validated following pairs:\n
{edge_pairs_tab}
' - self._send_msg(message, parse_mode=ParseMode.HTML) + if not edge_pairs: + message = 'Edge only validated following pairs:' + self._send_msg(message, parse_mode=ParseMode.HTML) + + for chunk in chunks(edge_pairs, 25): + edge_pairs_tab = tabulate(chunk, headers='keys', tablefmt='simple') + message = (f'Edge only validated following pairs:\n' + f'
{edge_pairs_tab}
') + + self._send_msg(message, parse_mode=ParseMode.HTML) + except RPCException as e: self._send_msg(str(e)) @@ -873,7 +1032,8 @@ class Telegram(RPCHandler): " `pending buy orders are marked with an asterisk (*)`\n" " `pending sell orders are marked with a double asterisk (**)`\n" "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n" - "*/profit:* `Lists cumulative profit from all finished trades`\n" + "*/profit []:* `Lists cumulative profit from all finished trades, " + "over the last n days`\n" "*/forcesell |all:* `Instantly sells the given trade or all trades, " "regardless of profit`\n" f"{forcebuy_text if self._config.get('forcebuy_enable', False) else ''}" @@ -884,6 +1044,7 @@ class Telegram(RPCHandler): "Avg. holding durationsfor buys and sells.`\n" "*/count:* `Show number of active trades compared to allowed number of trades`\n" "*/locks:* `Show currently locked pairs`\n" + "*/unlock :* `Unlock this Pair (or this lock id if it's numeric)`\n" "*/balance:* `Show account balance per currency`\n" "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" "*/reload_config:* `Reload configuration file` \n" @@ -945,12 +1106,15 @@ class Telegram(RPCHandler): f"*Current state:* `{val['state']}`" ) - def _update_msg(self, chat_id: str, message_id: str, msg: str, callback_path: str = "", reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None: + def _update_msg(self, chat_id: str, message_id: str, msg: str, callback_path: str = "", + reload_able: bool = False, parse_mode: str = ParseMode.MARKDOWN) -> None: if reload_able: - reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton("Refresh", callback_data=callback_path)]]) + reply_markup = InlineKeyboardMarkup([ + [InlineKeyboardButton("Refresh", callback_data=callback_path)], + ]) else: reply_markup = InlineKeyboardMarkup([[]]) - msg+="\nUpdated: {}".format(datetime.now().ctime()) + msg += "\nUpdated: {}".format(datetime.now().ctime()) try: try: self._updater.bot.edit_message_text( @@ -974,7 +1138,11 @@ class Telegram(RPCHandler): telegram_err.message ) - def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False, callback_path: str = "", reload_able: bool = False) -> None: + def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, + disable_notification: bool = False, + keyboard: List[List[Union[str, KeyboardButton, InlineKeyboardButton]]] = None, + callback_path: str = "", + reload_able: bool = False) -> None: """ Send given markdown message :param msg: message @@ -982,10 +1150,14 @@ class Telegram(RPCHandler): :param parse_mode: telegram parse mode :return: None """ - if reload_able and self._config['telegram'].get('reload',True): - reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton("Refresh", callback_data=callback_path)]]) + if reload_able and self._config['telegram'].get('reload', True): + reply_markup = InlineKeyboardMarkup([ + [InlineKeyboardButton("Refresh", callback_data=callback_path)]]) else: - reply_markup = ReplyKeyboardMarkup(self._keyboard, resize_keyboard=True) + if keyboard is not None: + reply_markup = InlineKeyboardMarkup(keyboard, resize_keyboard=True) + else: + reply_markup = ReplyKeyboardMarkup(self._keyboard, resize_keyboard=True) try: try: self._updater.bot.send_message( diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 5796201b5..0e4a4bf6f 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -6,7 +6,8 @@ from typing import Any, Dict from requests import RequestException, post -from freqtrade.rpc import RPC, RPCHandler, RPCMessageType +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPC, RPCHandler logger = logging.getLogger(__name__) @@ -28,6 +29,12 @@ class Webhook(RPCHandler): self._url = self._config['webhook']['url'] + self._format = self._config['webhook'].get('format', 'form') + + if self._format != 'form' and self._format != 'json': + raise NotImplementedError('Unknown webhook format `{}`, possible values are ' + '`form` (default) and `json`'.format(self._format)) + def cleanup(self) -> None: """ Cleanup pending module resources. @@ -39,17 +46,21 @@ class Webhook(RPCHandler): """ Send a message to telegram channel """ try: - if msg['type'] == RPCMessageType.BUY_NOTIFICATION: + if msg['type'] == RPCMessageType.BUY: valuedict = self._config['webhook'].get('webhookbuy', None) - elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: + elif msg['type'] == RPCMessageType.BUY_CANCEL: valuedict = self._config['webhook'].get('webhookbuycancel', None) - elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: + elif msg['type'] == RPCMessageType.BUY_FILL: + valuedict = self._config['webhook'].get('webhookbuyfill', None) + elif msg['type'] == RPCMessageType.SELL: valuedict = self._config['webhook'].get('webhooksell', None) - elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION: + elif msg['type'] == RPCMessageType.SELL_FILL: + valuedict = self._config['webhook'].get('webhooksellfill', None) + elif msg['type'] == RPCMessageType.SELL_CANCEL: valuedict = self._config['webhook'].get('webhooksellcancel', None) - elif msg['type'] in (RPCMessageType.STATUS_NOTIFICATION, - RPCMessageType.STARTUP_NOTIFICATION, - RPCMessageType.WARNING_NOTIFICATION): + elif msg['type'] in (RPCMessageType.STATUS, + RPCMessageType.STARTUP, + RPCMessageType.WARNING): valuedict = self._config['webhook'].get('webhookstatus', None) else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) @@ -66,7 +77,14 @@ class Webhook(RPCHandler): def _send_msg(self, payload: dict) -> None: """do the actual call to the webhook""" + if self._format == 'form': + kwargs = {'data': payload} + elif self._format == 'json': + kwargs = {'json': payload} + else: + raise NotImplementedError('Unknown format: {}'.format(self._format)) + try: - post(self._url, data=payload) + post(self._url, **kwargs) except RequestException as exc: logger.warning("Could not call webhook url. Exception: %s", exc) diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index 662156ae9..bd49165df 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -1,5 +1,7 @@ # flake8: noqa: F401 from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, timeframe_to_seconds) +from freqtrade.strategy.hyper import (CategoricalParameter, DecimalParameter, IntParameter, + RealParameter) from freqtrade.strategy.interface import IStrategy -from freqtrade.strategy.strategy_helper import merge_informative_pair +from freqtrade.strategy.strategy_helper import merge_informative_pair, stoploss_from_open diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py new file mode 100644 index 000000000..e487ffeff --- /dev/null +++ b/freqtrade/strategy/hyper.py @@ -0,0 +1,349 @@ +""" +IHyperStrategy interface, hyperoptable Parameter class. +This module defines a base class for auto-hyperoptable strategies. +""" +import logging +from abc import ABC, abstractmethod +from contextlib import suppress +from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union + +from freqtrade.optimize.hyperopt_tools import HyperoptTools + + +with suppress(ImportError): + from skopt.space import Integer, Real, Categorical + from freqtrade.optimize.space import SKDecimal + +from freqtrade.enums import RunMode +from freqtrade.exceptions import OperationalException + + +logger = logging.getLogger(__name__) + + +class BaseParameter(ABC): + """ + Defines a parameter that can be optimized by hyperopt. + """ + category: Optional[str] + default: Any + value: Any + in_space: bool = False + name: str + + def __init__(self, *, default: Any, space: Optional[str] = None, + optimize: bool = True, load: bool = True, **kwargs): + """ + Initialize hyperopt-optimizable parameter. + :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if + parameter field + name is prefixed with 'buy_' or 'sell_'. + :param optimize: Include parameter in hyperopt optimizations. + :param load: Load parameter value from {space}_params. + :param kwargs: Extra parameters to skopt.space.(Integer|Real|Categorical). + """ + if 'name' in kwargs: + raise OperationalException( + 'Name is determined by parameter field name and can not be specified manually.') + self.category = space + self._space_params = kwargs + self.value = default + self.optimize = optimize + self.load = load + + def __repr__(self): + return f'{self.__class__.__name__}({self.value})' + + @abstractmethod + def get_space(self, name: str) -> Union['Integer', 'Real', 'SKDecimal', 'Categorical']: + """ + Get-space - will be used by Hyperopt to get the hyperopt Space + """ + + +class NumericParameter(BaseParameter): + """ Internal parameter used for Numeric purposes """ + float_or_int = Union[int, float] + default: float_or_int + value: float_or_int + + def __init__(self, low: Union[float_or_int, Sequence[float_or_int]], + high: Optional[float_or_int] = None, *, default: float_or_int, + space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs): + """ + Initialize hyperopt-optimizable numeric parameter. + Cannot be instantiated, but provides the validation for other numeric parameters + :param low: Lower end (inclusive) of optimization space or [low, high]. + :param high: Upper end (inclusive) of optimization space. + Must be none of entire range is passed first parameter. + :param default: A default value. + :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if + parameter fieldname is prefixed with 'buy_' or 'sell_'. + :param optimize: Include parameter in hyperopt optimizations. + :param load: Load parameter value from {space}_params. + :param kwargs: Extra parameters to skopt.space.*. + """ + if high is not None and isinstance(low, Sequence): + raise OperationalException(f'{self.__class__.__name__} space invalid.') + if high is None or isinstance(low, Sequence): + if not isinstance(low, Sequence) or len(low) != 2: + raise OperationalException(f'{self.__class__.__name__} space must be [low, high]') + self.low, self.high = low + else: + self.low = low + self.high = high + + super().__init__(default=default, space=space, optimize=optimize, + load=load, **kwargs) + + +class IntParameter(NumericParameter): + default: int + value: int + + def __init__(self, low: Union[int, Sequence[int]], high: Optional[int] = None, *, default: int, + space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs): + """ + Initialize hyperopt-optimizable integer parameter. + :param low: Lower end (inclusive) of optimization space or [low, high]. + :param high: Upper end (inclusive) of optimization space. + Must be none of entire range is passed first parameter. + :param default: A default value. + :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if + parameter fieldname is prefixed with 'buy_' or 'sell_'. + :param optimize: Include parameter in hyperopt optimizations. + :param load: Load parameter value from {space}_params. + :param kwargs: Extra parameters to skopt.space.Integer. + """ + + super().__init__(low=low, high=high, default=default, space=space, optimize=optimize, + load=load, **kwargs) + + def get_space(self, name: str) -> 'Integer': + """ + Create skopt optimization space. + :param name: A name of parameter field. + """ + return Integer(low=self.low, high=self.high, name=name, **self._space_params) + + @property + def range(self): + """ + Get each value in this space as list. + Returns a List from low to high (inclusive) in Hyperopt mode. + Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid + calculating 100ds of indicators. + """ + if self.in_space and self.optimize: + # Scikit-optimize ranges are "inclusive", while python's "range" is exclusive + return range(self.low, self.high + 1) + else: + return range(self.value, self.value + 1) + + +class RealParameter(NumericParameter): + default: float + value: float + + def __init__(self, low: Union[float, Sequence[float]], high: Optional[float] = None, *, + default: float, space: Optional[str] = None, optimize: bool = True, + load: bool = True, **kwargs): + """ + Initialize hyperopt-optimizable floating point parameter with unlimited precision. + :param low: Lower end (inclusive) of optimization space or [low, high]. + :param high: Upper end (inclusive) of optimization space. + Must be none if entire range is passed first parameter. + :param default: A default value. + :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if + parameter fieldname is prefixed with 'buy_' or 'sell_'. + :param optimize: Include parameter in hyperopt optimizations. + :param load: Load parameter value from {space}_params. + :param kwargs: Extra parameters to skopt.space.Real. + """ + super().__init__(low=low, high=high, default=default, space=space, optimize=optimize, + load=load, **kwargs) + + def get_space(self, name: str) -> 'Real': + """ + Create skopt optimization space. + :param name: A name of parameter field. + """ + return Real(low=self.low, high=self.high, name=name, **self._space_params) + + +class DecimalParameter(NumericParameter): + default: float + value: float + + def __init__(self, low: Union[float, Sequence[float]], high: Optional[float] = None, *, + default: float, decimals: int = 3, space: Optional[str] = None, + optimize: bool = True, load: bool = True, **kwargs): + """ + Initialize hyperopt-optimizable decimal parameter with a limited precision. + :param low: Lower end (inclusive) of optimization space or [low, high]. + :param high: Upper end (inclusive) of optimization space. + Must be none if entire range is passed first parameter. + :param default: A default value. + :param decimals: A number of decimals after floating point to be included in testing. + :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if + parameter fieldname is prefixed with 'buy_' or 'sell_'. + :param optimize: Include parameter in hyperopt optimizations. + :param load: Load parameter value from {space}_params. + :param kwargs: Extra parameters to skopt.space.Integer. + """ + self._decimals = decimals + default = round(default, self._decimals) + + super().__init__(low=low, high=high, default=default, space=space, optimize=optimize, + load=load, **kwargs) + + def get_space(self, name: str) -> 'SKDecimal': + """ + Create skopt optimization space. + :param name: A name of parameter field. + """ + return SKDecimal(low=self.low, high=self.high, decimals=self._decimals, name=name, + **self._space_params) + + +class CategoricalParameter(BaseParameter): + default: Any + value: Any + opt_range: Sequence[Any] + + def __init__(self, categories: Sequence[Any], *, default: Optional[Any] = None, + space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs): + """ + Initialize hyperopt-optimizable parameter. + :param categories: Optimization space, [a, b, ...]. + :param default: A default value. If not specified, first item from specified space will be + used. + :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if + parameter field + name is prefixed with 'buy_' or 'sell_'. + :param optimize: Include parameter in hyperopt optimizations. + :param load: Load parameter value from {space}_params. + :param kwargs: Extra parameters to skopt.space.Categorical. + """ + if len(categories) < 2: + raise OperationalException( + 'CategoricalParameter space must be [a, b, ...] (at least two parameters)') + self.opt_range = categories + super().__init__(default=default, space=space, optimize=optimize, + load=load, **kwargs) + + def get_space(self, name: str) -> 'Categorical': + """ + Create skopt optimization space. + :param name: A name of parameter field. + """ + return Categorical(self.opt_range, name=name, **self._space_params) + + +class HyperStrategyMixin(object): + """ + A helper base class which allows HyperOptAuto class to reuse implementations of of buy/sell + strategy logic. + """ + + def __init__(self, config: Dict[str, Any], *args, **kwargs): + """ + Initialize hyperoptable strategy mixin. + """ + self.config = config + self.ft_buy_params: List[BaseParameter] = [] + self.ft_sell_params: List[BaseParameter] = [] + + self._load_hyper_params(config.get('runmode') == RunMode.HYPEROPT) + + def enumerate_parameters(self, category: str = None) -> Iterator[Tuple[str, BaseParameter]]: + """ + Find all optimizeable parameters and return (name, attr) iterator. + :param category: + :return: + """ + if category not in ('buy', 'sell', None): + raise OperationalException('Category must be one of: "buy", "sell", None.') + + if category is None: + params = self.ft_buy_params + self.ft_sell_params + else: + params = getattr(self, f"ft_{category}_params") + + for par in params: + yield par.name, par + + @classmethod + def detect_parameters(cls, category: str) -> Iterator[Tuple[str, BaseParameter]]: + """ Detect all parameters for 'category' """ + for attr_name in dir(cls): + if not attr_name.startswith('__'): # Ignore internals, not strictly necessary. + attr = getattr(cls, attr_name) + if issubclass(attr.__class__, BaseParameter): + if (attr_name.startswith(category + '_') + and attr.category is not None and attr.category != category): + raise OperationalException( + f'Inconclusive parameter name {attr_name}, category: {attr.category}.') + if (category == attr.category or + (attr_name.startswith(category + '_') and attr.category is None)): + yield attr_name, attr + + @classmethod + def detect_all_parameters(cls) -> Dict: + """ Detect all parameters and return them as a list""" + params: Dict = { + 'buy': list(cls.detect_parameters('buy')), + 'sell': list(cls.detect_parameters('sell')), + } + params.update({ + 'count': len(params['buy'] + params['sell']) + }) + + return params + + def _load_hyper_params(self, hyperopt: bool = False) -> None: + """ + Load Hyperoptable parameters + """ + self._load_params(getattr(self, 'buy_params', None), 'buy', hyperopt) + self._load_params(getattr(self, 'sell_params', None), 'sell', hyperopt) + + def _load_params(self, params: dict, space: str, hyperopt: bool = False) -> None: + """ + Set optimizeable parameter values. + :param params: Dictionary with new parameter values. + """ + if not params: + logger.info(f"No params for {space} found, using default values.") + param_container: List[BaseParameter] = getattr(self, f"ft_{space}_params") + + for attr_name, attr in self.detect_parameters(space): + attr.name = attr_name + attr.in_space = hyperopt and HyperoptTools.has_space(self.config, space) + if not attr.category: + attr.category = space + + param_container.append(attr) + + if params and attr_name in params: + if attr.load: + attr.value = params[attr_name] + logger.info(f'Strategy Parameter: {attr_name} = {attr.value}') + else: + logger.warning(f'Parameter "{attr_name}" exists, but is disabled. ' + f'Default value "{attr.value}" used.') + else: + logger.info(f'Strategy Parameter(default): {attr_name} = {attr.value}') + + def get_params_dict(self): + """ + Returns list of Parameters that are not part of the current optimize job + """ + params = { + 'buy': {}, + 'sell': {} + } + for name, p in self.enumerate_parameters(): + if not p.optimize or not p.in_space: + params[p.category][name] = p.value + return params diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index da4ce6c50..8ea38f503 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,60 +6,44 @@ import logging import warnings from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone -from enum import Enum -from typing import Dict, List, NamedTuple, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Union import arrow from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import SellType, SignalType from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.persistence import PairLocks, Trade +from freqtrade.strategy.hyper import HyperStrategyMixin from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) +CUSTOM_SELL_MAX_LENGTH = 64 -class SignalType(Enum): - """ - Enum to distinguish between buy and sell signals - """ - BUY = "buy" - SELL = "sell" - - -class SellType(Enum): - """ - Enum to distinguish between sell reasons - """ - ROI = "roi" - STOP_LOSS = "stop_loss" - STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange" - TRAILING_STOP_LOSS = "trailing_stop_loss" - SELL_SIGNAL = "sell_signal" - FORCE_SELL = "force_sell" - EMERGENCY_SELL = "emergency_sell" - NONE = "" - - def __str__(self): - # explicitly convert to String to help with exporting data. - return self.value - - -class SellCheckTuple(NamedTuple): +class SellCheckTuple(object): """ NamedTuple for Sell type + reason """ - sell_flag: bool sell_type: SellType + sell_reason: str = '' + + def __init__(self, sell_type: SellType, sell_reason: str = ''): + self.sell_type = sell_type + self.sell_reason = sell_reason or sell_type.value + + @property + def sell_flag(self): + return self.sell_type != SellType.NONE -class IStrategy(ABC): +class IStrategy(ABC, HyperStrategyMixin): """ Interface for freqtrade strategies Defines the mandatory structure must follow any custom strategies @@ -140,6 +124,7 @@ class IStrategy(ABC): self.config = config # Dict to determine if analysis is necessary self._last_candle_seen_per_pair: Dict[str, datetime] = {} + super().__init__(config) @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -149,6 +134,7 @@ class IStrategy(ABC): :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ + return dataframe @abstractmethod def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -158,6 +144,7 @@ class IStrategy(ABC): :param metadata: Additional information, like the currently traded pair :return: DataFrame with buy column """ + return dataframe @abstractmethod def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -167,6 +154,7 @@ class IStrategy(ABC): :param metadata: Additional information, like the currently traded pair :return: DataFrame with sell column """ + return dataframe def check_buy_timeout(self, pair: str, trade: Trade, order: dict, **kwargs) -> bool: """ @@ -214,7 +202,7 @@ class IStrategy(ABC): pass def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, **kwargs) -> bool: + time_in_force: str, current_time: datetime, **kwargs) -> bool: """ Called right before placing a buy order. Timing for this function is critical, so avoid doing heavy computations or @@ -229,6 +217,7 @@ class IStrategy(ABC): :param amount: Amount in target (quote) currency that's going to be traded. :param rate: Rate that's going to be used when using limit orders :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the buy-order is placed on the exchange. False aborts the process @@ -236,7 +225,8 @@ class IStrategy(ABC): return True def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float, - rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool: + rate: float, time_in_force: str, sell_reason: str, + current_time: datetime, **kwargs) -> bool: """ Called right before placing a regular sell order. Timing for this function is critical, so avoid doing heavy computations or @@ -255,6 +245,7 @@ class IStrategy(ABC): :param sell_reason: Sell reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', 'sell_signal', 'force_sell', 'emergency_sell'] + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the sell-order is placed on the exchange. False aborts the process @@ -283,6 +274,30 @@ class IStrategy(ABC): """ return self.stoploss + def custom_sell(self, pair: str, trade: Trade, current_time: datetime, current_rate: float, + current_profit: float, **kwargs) -> Optional[Union[str, bool]]: + """ + Custom sell signal logic indicating that specified position should be sold. Returning a + string or True from this method is equal to setting sell signal on a candle at specified + time. This method is not called when sell signal is set. + + This method should be overridden to create sell signals that depend on trade parameters. For + example you could implement a stoploss relative to candle when trade was opened, or a custom + 1:2 risk-reward ROI. + + Custom sell reason max length is 64. Exceeding this limit will raise OperationalException. + + :param pair: Pair that's currently analyzed + :param trade: trade object. + :param current_time: datetime object, containing the current datetime + :param current_rate: Rate, calculated based on pricing settings in ask_strategy. + :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return: To execute sell, return a string with custom sell reason or True. Otherwise return + None or False. + """ + return None + def informative_pairs(self) -> ListPairsWithTimeframes: """ Define additional, informative pair/interval combinations to be cached from the exchange. @@ -529,12 +544,33 @@ class IStrategy(ABC): and self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date)) + sell_signal = SellType.NONE + custom_reason = '' + # use provided rate in backtesting, not high/low. + current_rate = rate + current_profit = trade.calc_profit_ratio(current_rate) + if (ask_strategy.get('sell_profit_only', False) and current_profit <= ask_strategy.get('sell_profit_offset', 0)): # sell_profit_only and profit doesn't reach the offset - ignore sell signal - sell_signal = False - else: - sell_signal = sell and not buy and ask_strategy.get('use_sell_signal', True) + pass + elif ask_strategy.get('use_sell_signal', True) and not buy: + if sell: + sell_signal = SellType.SELL_SIGNAL + else: + custom_reason = strategy_safe_wrapper(self.custom_sell, default_retval=False)( + pair=trade.pair, trade=trade, current_time=date, current_rate=current_rate, + current_profit=current_profit) + if custom_reason: + sell_signal = SellType.CUSTOM_SELL + if isinstance(custom_reason, str): + if len(custom_reason) > CUSTOM_SELL_MAX_LENGTH: + logger.warning(f'Custom sell reason returned from custom_sell is too ' + f'long and was trimmed to {CUSTOM_SELL_MAX_LENGTH} ' + f'characters.') + custom_reason = custom_reason[:CUSTOM_SELL_MAX_LENGTH] + else: + custom_reason = None # TODO: return here if sell-signal should be favored over ROI # Start evaluations @@ -543,24 +579,23 @@ class IStrategy(ABC): # Sell-signal # Stoploss if roi_reached and stoplossflag.sell_type != SellType.STOP_LOSS: - logger.debug(f"{trade.pair} - Required profit reached. sell_flag=True, " - f"sell_type=SellType.ROI") - return SellCheckTuple(sell_flag=True, sell_type=SellType.ROI) + logger.debug(f"{trade.pair} - Required profit reached. sell_type=SellType.ROI") + return SellCheckTuple(sell_type=SellType.ROI) - if sell_signal: - logger.debug(f"{trade.pair} - Sell signal received. sell_flag=True, " - f"sell_type=SellType.SELL_SIGNAL") - return SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL) + if sell_signal != SellType.NONE: + logger.debug(f"{trade.pair} - Sell signal received. " + f"sell_type=SellType.{sell_signal.name}" + + (f", custom_reason={custom_reason}" if custom_reason else "")) + return SellCheckTuple(sell_type=sell_signal, sell_reason=custom_reason) if stoplossflag.sell_flag: - logger.debug(f"{trade.pair} - Stoploss hit. sell_flag=True, " - f"sell_type={stoplossflag.sell_type}") + logger.debug(f"{trade.pair} - Stoploss hit. sell_type={stoplossflag.sell_type}") return stoplossflag # This one is noisy, commented out... - # logger.debug(f"{trade.pair} - No sell signal. sell_flag=False") - return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) + # logger.debug(f"{trade.pair} - No sell signal.") + return SellCheckTuple(sell_type=SellType.NONE) def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, current_profit: float, @@ -624,9 +659,9 @@ class IStrategy(ABC): logger.debug(f"{trade.pair} - Trailing stop saved " f"{trade.stop_loss - trade.initial_stop_loss:.6f}") - return SellCheckTuple(sell_flag=True, sell_type=sell_type) + return SellCheckTuple(sell_type=sell_type) - return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) + return SellCheckTuple(sell_type=SellType.NONE) def min_roi_reached_entry(self, trade_dur: int) -> Tuple[Optional[int], Optional[float]]: """ @@ -649,7 +684,7 @@ class IStrategy(ABC): :return: True if bot should sell at current rate """ # Check if time matches and current rate is above threshold - trade_dur = int((current_time.timestamp() - trade.open_date.timestamp()) // 60) + trade_dur = int((current_time.timestamp() - trade.open_date_utc.timestamp()) // 60) _, roi = self.min_roi_reached_entry(trade_dur) if roi is None: return False @@ -659,7 +694,7 @@ class IStrategy(ABC): def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]: """ Populates indicators for given candle (OHLCV) data (for multiple pairs) - Does not run advice_buy or advise_sell! + Does not run advise_buy or advise_sell! Used by optimize operations only, not during dry / live runs. Using .copy() to get a fresh copy of the dataframe for every strategy run. Has positive effects on memory usage for whatever reason - also when diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index d7b1327d9..22b6f0be5 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -56,3 +56,30 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, dataframe = dataframe.ffill() return dataframe + + +def stoploss_from_open(open_relative_stop: float, current_profit: float) -> float: + """ + + Given the current profit, and a desired stop loss value relative to the open price, + return a stop loss value that is relative to the current price, and which can be + returned from `custom_stoploss`. + + The requested stop can be positive for a stop above the open price, or negative for + a stop below the open price. The return value is always >= 0. + + Returns 0 if the resulting stop price would be above the current price. + + :param open_relative_stop: Desired stop loss percentage relative to open price + :param current_profit: The current profit percentage + :return: Positive stop loss value relative to current price + """ + + # formula is undefined for current_profit -1, return maximum value + if current_profit == -1: + return 1 + + stoploss = 1-((1+open_relative_stop)/(1+current_profit)) + + # negative stoploss values indicate the requested stop price is higher than the current price + return max(stoploss, 0.0) diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index f920843b2..8933ebc6a 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -9,7 +9,8 @@ "cancel_open_orders_on_exit": false, "unfilledtimeout": { "buy": 10, - "sell": 30 + "sell": 30, + "unit": "minutes" }, "bid_strategy": { "price_side": "bid", @@ -54,14 +55,15 @@ "chat_id": "{{ telegram_chat_id }}" }, "api_server": { - "enabled": false, - "listen_ip_address": "127.0.0.1", + "enabled": {{ api_server | lower }}, + "listen_ip_address": "{{ api_server_listen_addr | default("127.0.0.1", true) }}", "listen_port": 8080, - "verbosity": "info", - "jwt_secret_key": "somethingrandom", + "verbosity": "error", + "enable_openapi": false, + "jwt_secret_key": "{{ api_server_jwt_key }}", "CORS_origins": [], - "username": "", - "password": "" + "username": "{{ api_server_username }}", + "password": "{{ api_server_password }}" }, "bot_name": "freqtrade", "initial_state": "running", diff --git a/freqtrade/templates/base_hyperopt.py.j2 b/freqtrade/templates/base_hyperopt.py.j2 index ec787cbb6..f6ca1477a 100644 --- a/freqtrade/templates/base_hyperopt.py.j2 +++ b/freqtrade/templates/base_hyperopt.py.j2 @@ -39,6 +39,15 @@ class {{ hyperopt }}(IHyperOpt): https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. """ + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching buy strategy parameters. + """ + return [ + {{ buy_space | indent(12) }} + ] + @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ @@ -79,12 +88,12 @@ class {{ hyperopt }}(IHyperOpt): return populate_buy_trend @staticmethod - def indicator_space() -> List[Dimension]: + def sell_indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching buy strategy parameters. + Define your Hyperopt space for searching sell strategy parameters. """ return [ - {{ buy_space | indent(12) }} + {{ sell_space | indent(12) }} ] @staticmethod @@ -126,11 +135,3 @@ class {{ hyperopt }}(IHyperOpt): return populate_sell_trend - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - {{ sell_space | indent(12) }} - ] diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index dd6b773e1..13fc0853a 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -1,4 +1,5 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement +# flake8: noqa: F401 # --- Do not remove these libs --- import numpy as np # noqa @@ -6,6 +7,7 @@ import pandas as pd # noqa from pandas import DataFrame from freqtrade.strategy import IStrategy +from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter # -------------------------------- # Add your lib to import here @@ -16,7 +18,7 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib class {{ strategy }}(IStrategy): """ This is a strategy template to get you started. - More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md + More information in https://www.freqtrade.io/en/latest/strategy-customization/ You can: :return: a Dataframe with all mandatory indicators for the strategies @@ -26,8 +28,9 @@ class {{ strategy }}(IStrategy): You must keep: - the lib in the section "Do not remove these libs" - - the prototype for the methods: minimal_roi, stoploss, populate_indicators, populate_buy_trend, - populate_sell_trend, hyperopt_space, buy_strategy_generator + - the methods: populate_indicators, populate_buy_trend, populate_sell_trend + You should keep: + - timeframe, minimal_roi, stoploss, trailing_* """ # Strategy interface version - allow new iterations of the strategy interface. # Check the documentation or the Sample strategy to get the latest version. diff --git a/freqtrade/templates/sample_hyperopt.py b/freqtrade/templates/sample_hyperopt.py index 10743e911..ed1af7718 100644 --- a/freqtrade/templates/sample_hyperopt.py +++ b/freqtrade/templates/sample_hyperopt.py @@ -45,6 +45,23 @@ class SampleHyperOpt(IHyperOpt): https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. """ + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching buy strategy parameters. + """ + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] + @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ @@ -92,20 +109,22 @@ class SampleHyperOpt(IHyperOpt): return populate_buy_trend @staticmethod - def indicator_space() -> List[Dimension]: + def sell_indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching buy strategy parameters. + Define your Hyperopt space for searching sell strategy parameters. """ return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + Integer(75, 100, name='sell-mfi-value'), + Integer(50, 100, name='sell-fastd-value'), + Integer(50, 100, name='sell-adx-value'), + Integer(60, 100, name='sell-rsi-value'), + Categorical([True, False], name='sell-mfi-enabled'), + Categorical([True, False], name='sell-fastd-enabled'), + Categorical([True, False], name='sell-adx-enabled'), + Categorical([True, False], name='sell-rsi-enabled'), + Categorical(['sell-bb_upper', + 'sell-macd_cross_signal', + 'sell-sar_reversal'], name='sell-trigger') ] @staticmethod @@ -153,56 +172,3 @@ class SampleHyperOpt(IHyperOpt): return dataframe return populate_sell_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. Should be a copy of same method from strategy. - Must align to populate_indicators in this file. - Only used when --spaces does not include buy space. - """ - dataframe.loc[ - ( - (dataframe['close'] < dataframe['bb_lowerband']) & - (dataframe['mfi'] < 16) & - (dataframe['adx'] > 25) & - (dataframe['rsi'] < 21) - ), - 'buy'] = 1 - - return dataframe - - def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. Should be a copy of same method from strategy. - Must align to populate_indicators in this file. - Only used when --spaces does not include sell space. - """ - dataframe.loc[ - ( - (qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) & - (dataframe['fastd'] > 54) - ), - 'sell'] = 1 - - return dataframe diff --git a/freqtrade/templates/sample_hyperopt_advanced.py b/freqtrade/templates/sample_hyperopt_advanced.py index 52e397466..cc13b6ba3 100644 --- a/freqtrade/templates/sample_hyperopt_advanced.py +++ b/freqtrade/templates/sample_hyperopt_advanced.py @@ -7,7 +7,7 @@ from typing import Any, Callable, Dict, List import numpy as np # noqa import pandas as pd # noqa from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer, Real # noqa +from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real # noqa from freqtrade.optimize.hyperopt_interface import IHyperOpt @@ -60,6 +60,23 @@ class AdvancedSampleHyperOpt(IHyperOpt): dataframe['sar'] = ta.SAR(dataframe) return dataframe + @staticmethod + def indicator_space() -> List[Dimension]: + """ + Define your Hyperopt space for searching buy strategy parameters. + """ + return [ + Integer(10, 25, name='mfi-value'), + Integer(15, 45, name='fastd-value'), + Integer(20, 50, name='adx-value'), + Integer(20, 40, name='rsi-value'), + Categorical([True, False], name='mfi-enabled'), + Categorical([True, False], name='fastd-enabled'), + Categorical([True, False], name='adx-enabled'), + Categorical([True, False], name='rsi-enabled'), + Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + ] + @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: """ @@ -106,20 +123,22 @@ class AdvancedSampleHyperOpt(IHyperOpt): return populate_buy_trend @staticmethod - def indicator_space() -> List[Dimension]: + def sell_indicator_space() -> List[Dimension]: """ - Define your Hyperopt space for searching strategy parameters + Define your Hyperopt space for searching sell strategy parameters. """ return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') + Integer(75, 100, name='sell-mfi-value'), + Integer(50, 100, name='sell-fastd-value'), + Integer(50, 100, name='sell-adx-value'), + Integer(60, 100, name='sell-rsi-value'), + Categorical([True, False], name='sell-mfi-enabled'), + Categorical([True, False], name='sell-fastd-enabled'), + Categorical([True, False], name='sell-adx-enabled'), + Categorical([True, False], name='sell-rsi-enabled'), + Categorical(['sell-bb_upper', + 'sell-macd_cross_signal', + 'sell-sar_reversal'], name='sell-trigger') ] @staticmethod @@ -168,25 +187,6 @@ class AdvancedSampleHyperOpt(IHyperOpt): return populate_sell_trend - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - @staticmethod def generate_roi_table(params: Dict) -> Dict[int, float]: """ @@ -223,9 +223,9 @@ class AdvancedSampleHyperOpt(IHyperOpt): Integer(10, 120, name='roi_t1'), Integer(10, 60, name='roi_t2'), Integer(10, 40, name='roi_t3'), - Real(0.01, 0.04, name='roi_p1'), - Real(0.01, 0.07, name='roi_p2'), - Real(0.01, 0.20, name='roi_p3'), + SKDecimal(0.01, 0.04, decimals=3, name='roi_p1'), + SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'), + SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'), ] @staticmethod @@ -237,7 +237,7 @@ class AdvancedSampleHyperOpt(IHyperOpt): 'stoploss' optimization hyperspace. """ return [ - Real(-0.35, -0.02, name='stoploss'), + SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'), ] @staticmethod @@ -256,51 +256,14 @@ class AdvancedSampleHyperOpt(IHyperOpt): # other 'trailing' hyperspace parameters. Categorical([True], name='trailing_stop'), - Real(0.01, 0.35, name='trailing_stop_positive'), + SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'), # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive', # so this intermediate parameter is used as the value of the difference between # them. The value of the 'trailing_stop_positive_offset' is constructed in the # generate_trailing_params() method. # This is similar to the hyperspace dimensions used for constructing the ROI tables. - Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'), + SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'), Categorical([True, False], name='trailing_only_offset_is_reached'), ] - - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. - Can be a copy of the corresponding method from the strategy, - or will be loaded from the strategy. - Must align to populate_indicators used (either from this File, or from the strategy) - Only used when --spaces does not include buy - """ - dataframe.loc[ - ( - (dataframe['close'] < dataframe['bb_lowerband']) & - (dataframe['mfi'] < 16) & - (dataframe['adx'] > 25) & - (dataframe['rsi'] < 21) - ), - 'buy'] = 1 - - return dataframe - - def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. - Can be a copy of the corresponding method from the strategy, - or will be loaded from the strategy. - Must align to populate_indicators used (either from this File, or from the strategy) - Only used when --spaces does not include sell - """ - dataframe.loc[ - ( - (qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) & - (dataframe['fastd'] > 54) - ), - 'sell'] = 1 - return dataframe diff --git a/freqtrade/templates/sample_hyperopt_loss.py b/freqtrade/templates/sample_hyperopt_loss.py index a2b28f948..343349508 100644 --- a/freqtrade/templates/sample_hyperopt_loss.py +++ b/freqtrade/templates/sample_hyperopt_loss.py @@ -1,5 +1,6 @@ from datetime import datetime from math import exp +from typing import Dict from pandas import DataFrame @@ -35,6 +36,7 @@ class SampleHyperOptLoss(IHyperOptLoss): @staticmethod def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, + config: Dict, processed: Dict[str, DataFrame], *args, **kwargs) -> float: """ Objective function, returns smaller number for better results diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index db1ba48b8..282b2f8e2 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -1,11 +1,13 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement +# flake8: noqa: F401 # isort: skip_file # --- Do not remove these libs --- import numpy as np # noqa import pandas as pd # noqa from pandas import DataFrame -from freqtrade.strategy.interface import IStrategy +from freqtrade.strategy import IStrategy +from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter # -------------------------------- # Add your lib to import here @@ -27,8 +29,9 @@ class SampleStrategy(IStrategy): You must keep: - the lib in the section "Do not remove these libs" - - the prototype for the methods: minimal_roi, stoploss, populate_indicators, populate_buy_trend, - populate_sell_trend, hyperopt_space, buy_strategy_generator + - the methods: populate_indicators, populate_buy_trend, populate_sell_trend + You should keep: + - timeframe, minimal_roi, stoploss, trailing_* """ # Strategy interface version - allow new iterations of the strategy interface. # Check the documentation or the Sample strategy to get the latest version. @@ -52,7 +55,11 @@ class SampleStrategy(IStrategy): # trailing_stop_positive = 0.01 # trailing_stop_positive_offset = 0.0 # Disabled / not configured - # Optimal ticker interval for the strategy. + # Hyperoptable parameters + buy_rsi = IntParameter(low=1, high=50, default=30, space='buy', optimize=True, load=True) + sell_rsi = IntParameter(low=50, high=100, default=70, space='sell', optimize=True, load=True) + + # Optimal timeframe for the strategy. timeframe = '5m' # Run "populate_indicators()" only for new candle. @@ -322,7 +329,7 @@ class SampleStrategy(IStrategy): """ # first check if dataprovider is available if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] @@ -339,7 +346,8 @@ class SampleStrategy(IStrategy): """ dataframe.loc[ ( - (qtpylib.crossed_above(dataframe['rsi'], 30)) & # Signal: RSI crosses above 30 + # Signal: RSI crosses above 30 + (qtpylib.crossed_above(dataframe['rsi'], self.buy_rsi.value)) & (dataframe['tema'] <= dataframe['bb_middleband']) & # Guard: tema below BB middle (dataframe['tema'] > dataframe['tema'].shift(1)) & # Guard: tema is raising (dataframe['volume'] > 0) # Make sure Volume is not 0 @@ -353,11 +361,12 @@ class SampleStrategy(IStrategy): Based on TA indicators, populates the sell signal for the given dataframe :param dataframe: DataFrame populated with indicators :param metadata: Additional information, like the currently traded pair - :return: DataFrame with buy column + :return: DataFrame with sell column """ dataframe.loc[ ( - (qtpylib.crossed_above(dataframe['rsi'], 70)) & # Signal: RSI crosses above 70 + # Signal: RSI crosses above 70 + (qtpylib.crossed_above(dataframe['rsi'], self.sell_rsi.value)) & (dataframe['tema'] > dataframe['bb_middleband']) & # Guard: tema above BB middle (dataframe['tema'] < dataframe['tema'].shift(1)) & # Guard: tema is falling (dataframe['volume'] > 0) # Make sure Volume is not 0 diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index c6e64c74e..0bc593e2d 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -40,7 +40,7 @@ "# Location of the data\n", "data_location = Path(config['user_data_dir'], 'data', 'binance')\n", "# Pair to analyze - Only use one pair here\n", - "pair = \"BTC_USDT\"" + "pair = \"BTC/USDT\"" ] }, { @@ -54,7 +54,9 @@ "\n", "candles = load_pair_history(datadir=data_location,\n", " timeframe=config[\"timeframe\"],\n", - " pair=pair)\n", + " pair=pair,\n", + " data_format = \"hdf5\",\n", + " )\n", "\n", "# Confirm success\n", "print(\"Loaded \" + str(len(candles)) + f\" rows of data for {pair} from {data_location}\")\n", @@ -280,6 +282,28 @@ "graph.show(renderer=\"browser\")\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot average profit per trade as distribution graph" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import plotly.figure_factory as ff\n", + "\n", + "hist_data = [trades.profit_ratio]\n", + "group_labels = ['profit_ratio'] # name of the dataset\n", + "\n", + "fig = ff.create_distplot(hist_data, group_labels,bin_size=0.01)\n", + "fig.show()\n" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 57d2ca665..a497b47cb 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -199,7 +199,7 @@ dataframe['htleadsine'] = hilbert['leadsine'] """ # first check if dataprovider is available if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] diff --git a/freqtrade/templates/subtemplates/indicators_minimal.j2 b/freqtrade/templates/subtemplates/indicators_minimal.j2 index 7d75b4610..90f4f4d4a 100644 --- a/freqtrade/templates/subtemplates/indicators_minimal.j2 +++ b/freqtrade/templates/subtemplates/indicators_minimal.j2 @@ -10,7 +10,7 @@ dataframe['rsi'] = ta.RSI(dataframe) """ # first check if dataprovider is available if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 53ededa19..2a9ac0690 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -14,8 +14,9 @@ def bot_loop_start(self, **kwargs) -> None: use_custom_stoploss = True -def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float, - current_profit: float, **kwargs) -> float: +def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', + current_rate: float, current_profit: float, dataframe: DataFrame, + **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -31,13 +32,14 @@ def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', c :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in ask_strategy. :param current_profit: Current profit (as ratio), calculated based on current_rate. + :param dataframe: Analyzed dataframe for this pair. Can contain future data in backtesting. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return float: New stoploss value, relative to the currentrate """ return self.stoploss def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, - time_in_force: str, **kwargs) -> bool: + time_in_force: str, current_time: 'datetime', **kwargs) -> bool: """ Called right before placing a buy order. Timing for this function is critical, so avoid doing heavy computations or @@ -52,6 +54,7 @@ def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: f :param amount: Amount in target (quote) currency that's going to be traded. :param rate: Rate that's going to be used when using limit orders :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the buy-order is placed on the exchange. False aborts the process @@ -59,7 +62,8 @@ def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: f return True def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float, - rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool: + rate: float, time_in_force: str, sell_reason: str, + current_time: 'datetime', **kwargs) -> bool: """ Called right before placing a regular sell order. Timing for this function is critical, so avoid doing heavy computations or @@ -78,6 +82,7 @@ def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: :param sell_reason: Sell reason. Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', 'sell_signal', 'force_sell', 'emergency_sell'] + :param current_time: datetime object, containing the current datetime :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. :return bool: When True is returned, then the sell-order is placed on the exchange. False aborts the process diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index d7dcfd487..1b2ec4550 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -8,9 +8,10 @@ from typing import Any, Dict, NamedTuple import arrow from freqtrade.constants import UNLIMITED_STAKE_AMOUNT +from freqtrade.enums import RunMode from freqtrade.exceptions import DependencyException from freqtrade.exchange import Exchange -from freqtrade.persistence import Trade +from freqtrade.persistence import LocalTrade, Trade logger = logging.getLogger(__name__) @@ -26,8 +27,9 @@ class Wallet(NamedTuple): class Wallets: - def __init__(self, config: dict, exchange: Exchange) -> None: + def __init__(self, config: dict, exchange: Exchange, log: bool = True) -> None: self._config = config + self._log = log self._exchange = exchange self._wallets: Dict[str, Wallet] = {} self.start_cap = config['dry_run_wallet'] @@ -64,9 +66,15 @@ class Wallets: """ # Recreate _wallets to reset closed trade balances _wallets = {} - closed_trades = Trade.get_trades(Trade.is_open.is_(False)).all() - open_trades = Trade.get_trades(Trade.is_open.is_(True)).all() - tot_profit = sum([trade.calc_profit() for trade in closed_trades]) + open_trades = Trade.get_trades_proxy(is_open=True) + # If not backtesting... + # TODO: potentially remove the ._log workaround to determine backtest mode. + if self._log: + closed_trades = Trade.get_trades_proxy(is_open=False) + tot_profit = sum( + [trade.close_profit_abs for trade in closed_trades if trade.close_profit_abs]) + else: + tot_profit = LocalTrade.total_profit tot_in_trades = sum([trade.stake_amount for trade in open_trades]) current_stake = self.start_cap + tot_profit - tot_in_trades @@ -91,12 +99,13 @@ class Wallets: balances = self._exchange.get_balances() for currency in balances: - self._wallets[currency] = Wallet( - currency, - balances[currency].get('free', None), - balances[currency].get('used', None), - balances[currency].get('total', None) - ) + if isinstance(balances[currency], dict): + self._wallets[currency] = Wallet( + currency, + balances[currency].get('free', None), + balances[currency].get('used', None), + balances[currency].get('total', None) + ) # Remove currencies no longer in get_balances output for currency in deepcopy(self._wallets): if currency not in balances: @@ -111,24 +120,24 @@ class Wallets: :param require_update: Allow skipping an update if balances were recently refreshed """ if (require_update or (self._last_wallet_refresh + 3600 < arrow.utcnow().int_timestamp)): - if self._config['dry_run']: - self._update_dry() - else: + if (not self._config['dry_run'] or self._config.get('runmode') == RunMode.LIVE): self._update_live() - logger.info('Wallets synced.') + else: + self._update_dry() + if self._log: + logger.info('Wallets synced.') self._last_wallet_refresh = arrow.utcnow().int_timestamp def get_all_balances(self) -> Dict[str, Any]: return self._wallets - def _get_available_stake_amount(self) -> float: + def _get_available_stake_amount(self, val_tied_up: float) -> float: """ Return the total currently available balance in stake currency, respecting tradable_balance_ratio. Calculated as - ( + free amount ) * tradable_balance_ratio - + ( + free amount) * tradable_balance_ratio - """ - val_tied_up = Trade.total_open_trades_stakes() # Ensure % is used from the overall balance # Otherwise we'd risk lowering stakes with each open trade. @@ -137,25 +146,26 @@ class Wallets: self._config['tradable_balance_ratio']) - val_tied_up return available_amount - def _calculate_unlimited_stake_amount(self, free_open_trades: int) -> float: + def _calculate_unlimited_stake_amount(self, available_amount: float, + val_tied_up: float) -> float: """ Calculate stake amount for "unlimited" stake amount :return: 0 if max number of trades reached, else stake_amount to use. """ - if not free_open_trades: + if self._config['max_open_trades'] == 0: return 0 - available_amount = self._get_available_stake_amount() + possible_stake = (available_amount + val_tied_up) / self._config['max_open_trades'] + # Theoretical amount can be above available amount - therefore limit to available amount! + return min(possible_stake, available_amount) - return available_amount / free_open_trades - - def _check_available_stake_amount(self, stake_amount: float) -> float: + def _check_available_stake_amount(self, stake_amount: float, available_amount: float) -> float: """ Check if stake amount can be fulfilled with the available balance for the stake currency :return: float: Stake amount + :raise: DependencyException if balance is lower than stake-amount """ - available_amount = self._get_available_stake_amount() if self._config['amend_last_stake_amount']: # Remaining amount needs to be at least stake_amount * last_stake_amount_min_ratio @@ -173,7 +183,7 @@ class Wallets: return stake_amount - def get_trade_stake_amount(self, pair: str, free_open_trades: int, edge=None) -> float: + def get_trade_stake_amount(self, pair: str, edge=None) -> float: """ Calculate stake amount for the trade :return: float: Stake amount @@ -182,17 +192,20 @@ class Wallets: stake_amount: float # Ensure wallets are uptodate. self.update() + val_tied_up = Trade.total_open_trades_stakes() + available_amount = self._get_available_stake_amount(val_tied_up) if edge: stake_amount = edge.stake_amount( pair, self.get_free(self._config['stake_currency']), self.get_total(self._config['stake_currency']), - Trade.total_open_trades_stakes() + val_tied_up ) else: stake_amount = self._config['stake_amount'] if stake_amount == UNLIMITED_STAKE_AMOUNT: - stake_amount = self._calculate_unlimited_stake_amount(free_open_trades) + stake_amount = self._calculate_unlimited_stake_amount( + available_amount, val_tied_up) - return self._check_available_stake_amount(stake_amount) + return self._check_available_stake_amount(stake_amount, available_amount) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index ec9331eef..4fa9166bf 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -11,9 +11,9 @@ import sdnotify from freqtrade import __version__, constants from freqtrade.configuration import Configuration +from freqtrade.enums import State from freqtrade.exceptions import OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.state import State logger = logging.getLogger(__name__) diff --git a/mkdocs.yml b/mkdocs.yml index 18fccc333..cc5747225 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,7 @@ site_name: Freqtrade +site_url: https://www.freqtrade.io/ +repo_url: https://github.com/freqtrade/freqtrade +use_directory_urls: True nav: - Home: index.md - Quickstart with Docker: docker_quickstart.md @@ -13,16 +16,16 @@ nav: - Start the bot: bot-usage.md - Control the bot: - Telegram: telegram-usage.md - - Web Hook: webhook-config.md - REST API & FreqUI: rest-api.md + - Web Hook: webhook-config.md - Data Downloading: data-download.md - Backtesting: backtesting.md - Hyperopt: hyperopt.md - Utility Sub-commands: utils.md + - Plotting: plotting.md - Data Analysis: - Jupyter Notebooks: data-analysis.md - Strategy analysis: strategy_analysis_example.md - - Plotting: plotting.md - Exchange-specific Notes: exchanges.md - Advanced Topics: - Advanced Post-installation Tasks: advanced-setup.md @@ -50,24 +53,25 @@ extra_javascript: - https://polyfill.io/v3/polyfill.min.js?features=es6 - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js markdown_extensions: - - admonition - - footnotes - - codehilite: - guess_lang: false - - toc: - permalink: true - - pymdownx.arithmatex: - generic: true - - pymdownx.details - - pymdownx.inlinehilite - - pymdownx.magiclink - - pymdownx.pathconverter - - pymdownx.smartsymbols - - pymdownx.snippets: - base_path: docs - check_paths: true - - pymdownx.tabbed - - pymdownx.superfences - - pymdownx.tasklist: - custom_checkbox: true - - mdx_truly_sane_lists + - attr_list + - admonition + - footnotes + - codehilite: + guess_lang: false + - toc: + permalink: true + - pymdownx.arithmatex: + generic: true + - pymdownx.details + - pymdownx.inlinehilite + - pymdownx.magiclink + - pymdownx.pathconverter + - pymdownx.smartsymbols + - pymdownx.snippets: + base_path: docs + check_paths: true + - pymdownx.tabbed + - pymdownx.superfences + - pymdownx.tasklist: + custom_checkbox: true + - mdx_truly_sane_lists diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..f0637d8c6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[tool.black] +line-length = 100 +exclude = ''' +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + )/ + # Exclude vendor directory + | vendor +) +''' + +[tool.isort] +line_length = 100 +multi_line_output=0 +lines_after_imports=2 + +[build-system] +requires = ["setuptools >= 46.4.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt index fa0ead603..6fbe581a5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,17 +3,17 @@ -r requirements-plot.txt -r requirements-hyperopt.txt -coveralls==3.0.0 -flake8==3.8.4 +coveralls==3.1.0 +flake8==3.9.2 flake8-type-annotations==0.1.0 -flake8-tidy-imports==4.2.1 -mypy==0.790 -pytest==6.2.2 -pytest-asyncio==0.14.0 -pytest-cov==2.11.1 -pytest-mock==3.5.1 +flake8-tidy-imports==4.3.0 +mypy==0.812 +pytest==6.2.4 +pytest-asyncio==0.15.1 +pytest-cov==2.12.1 +pytest-mock==3.6.1 pytest-random-order==1.0.4 -isort==5.7.0 +isort==5.8.0 # Convert jupyter notebooks to markdown documents nbconvert==6.0.7 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 104fbf454..5e7e9d9d2 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,9 +2,10 @@ -r requirements.txt # Required for hyperopt -scipy==1.6.0 -scikit-learn==0.24.1 +scipy==1.6.3 +scikit-learn==0.24.2 scikit-optimize==0.8.1 filelock==3.0.12 -joblib==1.0.0 +joblib==1.0.1 +psutil==5.8.0 progressbar2==3.53.1 diff --git a/requirements.txt b/requirements.txt index 5d03a9c2d..abfae55b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,24 +1,25 @@ -numpy==1.20.1 -pandas==1.2.1 +numpy==1.20.3 +pandas==1.2.4 -ccxt==1.41.70 +ccxt==1.51.3 # Pin cryptography for now due to rust build errors with piwheels -cryptography==3.3.2 -aiohttp==3.7.3 -SQLAlchemy==1.3.23 -python-telegram-bot==13.2 -arrow==0.17.0 -cachetools==4.2.1 +cryptography==3.4.7 +aiohttp==3.7.4.post0 +SQLAlchemy==1.4.17 +python-telegram-bot==13.6 +arrow==1.1.0 +cachetools==4.2.2 requests==2.25.1 -urllib3==1.26.3 +urllib3==1.26.5 wrapt==1.12.1 jsonschema==3.2.0 -TA-Lib==0.4.19 -tabulate==0.8.7 -pycoingecko==1.4.0 -jinja2==2.11.3 +TA-Lib==0.4.20 +technical==1.3.0 +tabulate==0.8.9 +pycoingecko==2.1.0 +jinja2==3.0.1 tables==3.6.1 -blosc==1.10.2 +blosc==1.10.4 # find first, C search in arrays py_find_1st==1.1.5 @@ -30,13 +31,13 @@ python-rapidjson==1.0 sdnotify==0.3.2 # API Server -fastapi==0.63.0 -uvicorn==0.13.3 -pyjwt==2.0.1 -aiofiles==0.6.0 +fastapi==0.65.2 +uvicorn==0.14.0 +pyjwt==2.1.0 +aiofiles==0.7.0 # Support for colorized terminal output colorama==0.4.4 # Building config files interactively questionary==1.9.0 -prompt-toolkit==3.0.14 +prompt-toolkit==3.0.18 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index b6e66cfa4..ece0a253e 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -118,8 +118,16 @@ class FtRestClient(): """ return self._get("locks") + def delete_lock(self, lock_id): + """Delete (disable) lock from the database. + + :param lock_id: ID for the lock to delete + :return: json object + """ + return self._delete("locks/{}".format(lock_id)) + def daily(self, days=None): - """Return the amount of open trades. + """Return the profits for each day, and amount of trades. :return: json object """ @@ -174,21 +182,45 @@ class FtRestClient(): """ return self._get("show_config") + def ping(self): + """simple ping""" + configstatus = self.show_config() + if not configstatus: + return {"status": "not_running"} + elif configstatus['state'] == "running": + return {"status": "pong"} + else: + return {"status": "not_running"} + def logs(self, limit=None): """Show latest logs. - :param limit: Limits log messages to the last logs. No limit to get all the trades. + :param limit: Limits log messages to the last logs. No limit to get the entire log. :return: json object """ return self._get("logs", params={"limit": limit} if limit else 0) - def trades(self, limit=None): - """Return trades history. + def trades(self, limit=None, offset=None): + """Return trades history, sorted by id - :param limit: Limits trades to the X last trades. No limit to get all the trades. + :param limit: Limits trades to the X last trades. Max 500 trades. + :param offset: Offset by this amount of trades. :return: json object """ - return self._get("trades", params={"limit": limit} if limit else 0) + params = {} + if limit: + params['limit'] = limit + if offset: + params['offset'] = offset + return self._get("trades", params) + + def trade(self, trade_id): + """Return specific trade + + :param trade_id: Specify which trade to get. + :return: json object + """ + return self._get("trade/{}".format(trade_id)) def delete_trade(self, trade_id): """Delete trade from the database. @@ -364,7 +396,7 @@ def main(args): sys.exit() config = load_config(args['config']) - url = config.get('api_server', {}).get('server_url', '127.0.0.1') + url = config.get('api_server', {}).get('listen_ip_address', '127.0.0.1') port = config.get('api_server', {}).get('listen_port', '8080') username = config.get('api_server', {}).get('username') password = config.get('api_server', {}).get('password') diff --git a/setup.cfg b/setup.cfg index be2cd450c..b311c94da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,43 @@ +[metadata] +name = freqtrade +version = attr: freqtrade.__version__ +author = Freqtrade Team +author_email = freqtrade@protonmail.com +description = Freqtrade - Crypto Trading Bot +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/freqtrade/freqtrade +project_urls = + Bug Tracker = https://github.com/freqtrade/freqtrade/issues +license = GPLv3 +classifiers = + Environment :: Console + Intended Audience :: Science/Research + License :: OSI Approved :: GNU General Public License v3 (GPLv3) + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Operating System :: MacOS + Operating System :: Unix + Topic :: Office/Business :: Financial :: Investment + + +[options] +zip_safe = False +include_package_data = True +tests_require = + pytest + pytest-asyncio + pytest-cov + pytest-mock + +packages = find: +python_requires = >=3.6 + +[options.entry_points] +console_scripts = + freqtrade = freqtrade.main:main + [flake8] #ignore = max-line-length = 100 @@ -8,11 +48,6 @@ exclude = .eggs, user_data, -[isort] -line_length=100 -multi_line_output=0 -lines_after_imports=2 - [mypy] ignore_missing_imports = True diff --git a/setup.py b/setup.py index 030980c96..727c40c7c 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,7 @@ -from sys import version_info - from setuptools import setup -if version_info.major == 3 and version_info.minor < 7 or \ - version_info.major < 3: - print('Your Python interpreter must be 3.7 or greater!') - exit(1) - -from pathlib import Path # noqa: E402 - -from freqtrade import __version__ # noqa: E402 - - -readme_file = Path(__file__).parent / "README.md" -readme_long = "Crypto Trading Bot" -if readme_file.is_file(): - readme_long = (Path(__file__).parent / "README.md").read_text() - # Requirements used for submodules -api = ['flask', 'flask-jwt-extended', 'flask-cors'] plot = ['plotly>=4.0'] hyperopt = [ 'scipy', @@ -28,6 +10,7 @@ hyperopt = [ 'filelock', 'joblib', 'progressbar2', + 'psutil', ] develop = [ @@ -50,68 +33,51 @@ jupyter = [ 'nbconvert', ] -all_extra = api + plot + develop + jupyter + hyperopt +all_extra = plot + develop + jupyter + hyperopt -setup(name='freqtrade', - version=__version__, - description='Crypto Trading Bot', - long_description=readme_long, - long_description_content_type="text/markdown", - url='https://github.com/freqtrade/freqtrade', - author='Freqtrade Team', - author_email='michael.egger@tsn.at', - license='GPLv3', - packages=['freqtrade'], - setup_requires=['pytest-runner', 'numpy'], - tests_require=['pytest', 'pytest-asyncio', 'pytest-cov', 'pytest-mock', ], - install_requires=[ - # from requirements.txt - 'ccxt>=1.24.96', - 'SQLAlchemy', - 'python-telegram-bot', - 'arrow>=0.17.0', - 'cachetools', - 'requests', - 'urllib3', - 'wrapt', - 'jsonschema', - 'TA-Lib', - 'tabulate', - 'pycoingecko', - 'py_find_1st', - 'python-rapidjson', - 'sdnotify', - 'colorama', - 'jinja2', - 'questionary', - 'prompt-toolkit', - 'numpy', - 'pandas', - 'tables', - 'blosc', - ], - extras_require={ - 'api': api, - 'dev': all_extra, - 'plot': plot, - 'jupyter': jupyter, - 'hyperopt': hyperopt, - 'all': all_extra, - }, - include_package_data=True, - zip_safe=False, - entry_points={ - 'console_scripts': [ - 'freqtrade = freqtrade.main:main', - ], - }, - classifiers=[ - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Operating System :: MacOS', - 'Operating System :: Unix', - 'Topic :: Office/Business :: Financial :: Investment', - ]) +setup( + tests_require=[ + 'pytest', + 'pytest-asyncio', + 'pytest-cov', + 'pytest-mock', + ], + install_requires=[ + # from requirements.txt + 'ccxt>=1.50.48', + 'SQLAlchemy', + 'python-telegram-bot>=13.4', + 'arrow>=0.17.0', + 'cachetools', + 'requests', + 'urllib3', + 'wrapt', + 'jsonschema', + 'TA-Lib', + 'technical', + 'tabulate', + 'pycoingecko', + 'py_find_1st', + 'python-rapidjson', + 'sdnotify', + 'colorama', + 'jinja2', + 'questionary', + 'prompt-toolkit', + 'numpy', + 'pandas', + 'tables', + 'blosc', + 'fastapi', + 'uvicorn', + 'pyjwt', + 'aiofiles' + ], + extras_require={ + 'dev': all_extra, + 'plot': plot, + 'jupyter': jupyter, + 'hyperopt': hyperopt, + 'all': all_extra, + }, +) diff --git a/setup.sh b/setup.sh index d0ca1f643..631c31df2 100755 --- a/setup.sh +++ b/setup.sh @@ -138,7 +138,7 @@ function install_macos() { # Install bot Debian_ubuntu function install_debian() { sudo apt-get update - sudo apt-get install -y build-essential autoconf libtool pkg-config make wget git + sudo apt-get install -y build-essential autoconf libtool pkg-config make wget git libpython3-dev install_talib } diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index 291720f4b..66c750e79 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -50,6 +50,10 @@ def test_start_new_config(mocker, caplog, exchange): 'telegram': False, 'telegram_token': 'asdf1244', 'telegram_chat_id': '1144444', + 'api_server': False, + 'api_server_listen_addr': '127.0.0.1', + 'api_server_username': 'freqtrader', + 'api_server_password': 'MoneyMachine', } mocker.patch('freqtrade.commands.build_config_commands.ask_user_config', return_value=sample_selections) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index c81909025..47f298ad7 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,3 +1,4 @@ +import json import re from io import BytesIO from pathlib import Path @@ -16,8 +17,8 @@ from freqtrade.commands import (start_convert_data, start_create_userdir, start_ from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) from tests.conftest_trades import MOCK_TRADE_COUNT @@ -66,8 +67,8 @@ def test_list_exchanges(capsys): start_list_exchanges(get_args(args)) captured = capsys.readouterr() assert re.match(r"Exchanges available for Freqtrade.*", captured.out) - assert re.match(r".*binance,.*", captured.out) - assert re.match(r".*bittrex,.*", captured.out) + assert re.search(r".*binance.*", captured.out) + assert re.search(r".*bittrex.*", captured.out) # Test with --one-column args = [ @@ -89,9 +90,9 @@ def test_list_exchanges(capsys): start_list_exchanges(get_args(args)) captured = capsys.readouterr() assert re.match(r"All exchanges supported by the ccxt library.*", captured.out) - assert re.match(r".*binance,.*", captured.out) - assert re.match(r".*bittrex,.*", captured.out) - assert re.match(r".*bitmex,.*", captured.out) + assert re.search(r".*binance.*", captured.out) + assert re.search(r".*bittrex.*", captured.out) + assert re.search(r".*bitmex.*", captured.out) # Test with --one-column --all args = [ @@ -116,7 +117,7 @@ def test_list_timeframes(mocker, capsys): '1h': 'hour', '1d': 'day', } - patch_exchange(mocker, api_mock=api_mock) + patch_exchange(mocker, api_mock=api_mock, id='bittrex') args = [ "list-timeframes", ] @@ -201,7 +202,7 @@ def test_list_markets(mocker, markets, capsys): api_mock = MagicMock() api_mock.markets = markets - patch_exchange(mocker, api_mock=api_mock) + patch_exchange(mocker, api_mock=api_mock, id='bittrex') # Test with no --config args = [ @@ -706,7 +707,7 @@ def test_download_data_timerange(mocker, caplog, markets): start_download_data(get_args(args)) assert dl_mock.call_count == 1 # 20days ago - days_ago = arrow.get(arrow.utcnow().shift(days=-20).date()).int_timestamp + days_ago = arrow.get(arrow.now().shift(days=-20).date()).int_timestamp assert dl_mock.call_args_list[0][1]['timerange'].startts == days_ago dl_mock.reset_mock() @@ -914,246 +915,258 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): ] start_test_pairlist(get_args(args)) captured = capsys.readouterr() - assert re.match(r'Pairs for BTC: \n\["ETH/BTC","TKN/BTC","BLK/BTC","LTC/BTC","XRP/BTC"\]\n', - captured.out) + try: + json_pairs = json.loads(captured.out) + assert 'ETH/BTC' in json_pairs + assert 'TKN/BTC' in json_pairs + assert 'BLK/BTC' in json_pairs + assert 'LTC/BTC' in json_pairs + assert 'XRP/BTC' in json_pairs + except json.decoder.JSONDecodeError: + pytest.fail(f'Expected well formed JSON, but failed to parse: {captured.out}') -def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): +def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, + saved_hyperopt_results_legacy, tmpdir): + csv_file = Path(tmpdir) / "test.csv" + for res in (saved_hyperopt_results, saved_hyperopt_results_legacy): + mocker.patch( + 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', + MagicMock(return_value=res) + ) + + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", + " 6/12", " 7/12", " 8/12", " 9/12", " 10/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--best", + "--no-details", + "--no-color", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 5/12", " 10/12"]) + assert all(x not in captured.out + for x in [" 2/12", " 3/12", " 4/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--no-color", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-color", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params", + "Sell hyperspace params", "ROI table", "Stoploss"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--min-trades", "20", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 3/12", " 6/12", " 7/12", " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 8/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--no-color", + "--max-trades", "20", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--no-color", + "--min-avg-profit", "0.11", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 10/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--max-avg-profit", "0.10", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12"]) + assert all(x not in captured.out + for x in [" 2/12", " 4/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--min-total-profit", "0.4", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--max-total-profit", "0.4", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 4/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--min-objective", "0.1", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--max-objective", "0.1", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 4/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--no-color", + "--min-avg-time", "2000", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", + " 8/12", " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--max-avg-time", "1500", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 6/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12" + " 9/12", " 10/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--export-csv", + str(csv_file), + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + log_has("CSV file created: test_file.csv", caplog) + assert csv_file.is_file() + line = csv_file.read_text() + assert ('Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' in line + or "Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,2 days 17:30:00,0.43662" in line) + csv_file.unlink() + + +def test_hyperopt_show(mocker, capsys, saved_hyperopt_results): mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.load_previous_results', - MagicMock(return_value=hyperopt_results) - ) - - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", - " 6/12", " 7/12", " 8/12", " 9/12", " 10/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--best", - "--no-details", - "--no-color", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 5/12", " 10/12"]) - assert all(x not in captured.out - for x in [" 2/12", " 3/12", " 4/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-details", - "--no-color", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12", " 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-color", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params", - "Sell hyperspace params", "ROI table", "Stoploss"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--min-trades", "20", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 3/12", " 6/12", " 7/12", " 9/12", " 11/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 8/12", " 10/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-details", - "--no-color", - "--max-trades", "20", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12", " 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-details", - "--no-color", - "--min-avg-profit", "0.11", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 10/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--max-avg-profit", "0.10", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", - " 11/12"]) - assert all(x not in captured.out - for x in [" 2/12", " 4/12", " 10/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--min-total-profit", "0.4", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", - " 9/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--max-total-profit", "0.4", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", - " 9/12", " 11/12"]) - assert all(x not in captured.out - for x in [" 4/12", " 10/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--min-objective", "0.1", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", - " 9/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--max-objective", "0.1", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", - " 9/12", " 11/12"]) - assert all(x not in captured.out - for x in [" 4/12", " 10/12", " 12/12"]) - args = [ - "hyperopt-list", - "--profitable", - "--no-details", - "--no-color", - "--min-avg-time", "2000", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 10/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", - " 8/12", " 9/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--max-avg-time", "1500", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - assert all(x in captured.out - for x in [" 2/12", " 6/12"]) - assert all(x not in captured.out - for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12" - " 9/12", " 10/12", " 11/12", " 12/12"]) - args = [ - "hyperopt-list", - "--no-details", - "--no-color", - "--export-csv", "test_file.csv", - ] - pargs = get_args(args) - pargs['config'] = None - start_hyperopt_list(pargs) - captured = capsys.readouterr() - log_has("CSV file created: test_file.csv", caplog) - f = Path("test_file.csv") - assert 'Best,1,2,-1.25%,-0.00125625,,-2.51,"3,930.0 m",0.43662' in f.read_text() - assert f.is_file() - f.unlink() - - -def test_hyperopt_show(mocker, capsys, hyperopt_results): - mocker.patch( - 'freqtrade.optimize.hyperopt.Hyperopt.load_previous_results', - MagicMock(return_value=hyperopt_results) + 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', + MagicMock(return_value=saved_hyperopt_results) ) args = [ diff --git a/tests/config_test_comments.json b/tests/config_test_comments.json index 4f201f86c..48a087dec 100644 --- a/tests/config_test_comments.json +++ b/tests/config_test_comments.json @@ -59,7 +59,7 @@ } }, "exchange": { - "name": "bittrex", + "name": "binance", "sandbox": false, "key": "your_exchange_key", "secret": "your_exchange_secret", diff --git a/tests/conftest.py b/tests/conftest.py index 61899dd53..dd38ca610 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,10 +3,10 @@ import json import logging import re from copy import deepcopy -from datetime import datetime +from datetime import datetime, timedelta from functools import reduce from pathlib import Path -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import MagicMock, Mock, PropertyMock import arrow import numpy as np @@ -17,9 +17,10 @@ from freqtrade import constants from freqtrade.commands import Arguments from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo +from freqtrade.enums import RunMode from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.persistence import Trade, init_db +from freqtrade.persistence import LocalTrade, Trade, init_db from freqtrade.resolvers import ExchangeResolver from freqtrade.worker import Worker from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4, @@ -64,6 +65,14 @@ def get_args(args): return Arguments(args).get_parsed_arg() +# Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines +def get_mock_coro(return_value): + async def mock_coro(*args, **kwargs): + return return_value + + return Mock(wraps=mock_coro) + + def patched_configuration_load_config_file(mocker, config) -> None: mocker.patch( 'freqtrade.configuration.configuration.load_config_file', @@ -71,7 +80,7 @@ def patched_configuration_load_config_file(mocker, config) -> None: ) -def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> None: +def patch_exchange(mocker, api_mock=None, id='binance', mock_markets=True) -> None: mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) @@ -90,7 +99,7 @@ def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> No mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock()) -def get_patched_exchange(mocker, config, api_mock=None, id='bittrex', +def get_patched_exchange(mocker, config, api_mock=None, id='binance', mock_markets=True) -> Exchange: patch_exchange(mocker, api_mock, id, mock_markets) config['exchange']['name'] = id @@ -183,28 +192,37 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: freqtrade.exchange.refresh_latest_ohlcv = lambda p: None -def create_mock_trades(fee): +def create_mock_trades(fee, use_db: bool = True): """ Create some fake trades ... """ + def add_trade(trade): + if use_db: + Trade.query.session.add(trade) + else: + LocalTrade.add_bt_trade(trade) + # Simulate dry_run entries trade = mock_trade_1(fee) - Trade.session.add(trade) + add_trade(trade) trade = mock_trade_2(fee) - Trade.session.add(trade) + add_trade(trade) trade = mock_trade_3(fee) - Trade.session.add(trade) + add_trade(trade) trade = mock_trade_4(fee) - Trade.session.add(trade) + add_trade(trade) trade = mock_trade_5(fee) - Trade.session.add(trade) + add_trade(trade) trade = mock_trade_6(fee) - Trade.session.add(trade) + add_trade(trade) + + if use_db: + Trade.query.session.flush() @pytest.fixture(autouse=True) @@ -255,6 +273,7 @@ def get_default_conf(testdatadir): "20": 0.02, "0": 0.04 }, + "dry_run_wallet": 1000, "stoploss": -0.10, "unfilledtimeout": { "buy": 10, @@ -275,7 +294,7 @@ def get_default_conf(testdatadir): "order_book_max": 1 }, "exchange": { - "name": "bittrex", + "name": "binance", "enabled": True, "key": "key", "secret": "secret", @@ -296,7 +315,8 @@ def get_default_conf(testdatadir): "telegram": { "enabled": True, "token": "token", - "chat_id": "0" + "chat_id": "0", + "notification_settings": {}, }, "datadir": str(testdatadir), "initial_state": "running", @@ -1658,6 +1678,7 @@ def buy_order_fee(): @pytest.fixture(scope="function") def edge_conf(default_conf): conf = deepcopy(default_conf) + conf['runmode'] = RunMode.DRY_RUN conf['max_open_trades'] = -1 conf['tradable_balance_ratio'] = 0.5 conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT @@ -1729,7 +1750,7 @@ def import_fails() -> None: realimport = builtins.__import__ def mockedimport(name, *args, **kwargs): - if name in ["filelock", 'systemd.journal']: + if name in ["filelock", 'systemd.journal', 'uvloop']: raise ImportError(f"No module named '{name}'") return realimport(name, *args, **kwargs) @@ -1747,7 +1768,7 @@ def open_trade(): return Trade( pair='ETH/BTC', open_rate=0.00001099, - exchange='bittrex', + exchange='binance', open_order_id='123456789', amount=90.99181073, fee_open=0.0, @@ -1759,14 +1780,14 @@ def open_trade(): @pytest.fixture -def hyperopt_results(): +def saved_hyperopt_results_legacy(): return [ { 'loss': 0.4366182531161519, 'params_dict': { 'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1190, 'roi_t2': 541, 'roi_t3': 408, 'roi_p1': 0.026035863879169705, 'roi_p2': 0.12508730043628782, 'roi_p3': 0.27766427921605896, 'stoploss': -0.2562930402099556}, # noqa: E501 'params_details': {'buy': {'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4287874435315165, 408: 0.15112316431545753, 949: 0.026035863879169705, 2139: 0}, 'stoploss': {'stoploss': -0.2562930402099556}}, # noqa: E501 - 'results_metrics': {'trade_count': 2, 'avg_profit': -1.254995, 'total_profit': -0.00125625, 'profit': -2.50999, 'duration': 3930.0}, # noqa: E501 + 'results_metrics': {'trade_count': 2, 'avg_profit': -1.254995, 'median_profit': -1.2222, 'total_profit': -0.00125625, 'profit': -2.50999, 'duration': 3930.0}, # noqa: E501 'results_explanation': ' 2 trades. Avg profit -1.25%. Total profit -0.00125625 BTC ( -2.51Σ%). Avg duration 3930.0 min.', # noqa: E501 'total_profit': -0.00125625, 'current_epoch': 1, @@ -1781,7 +1802,7 @@ def hyperopt_results(): 'sell': {'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, # noqa: E501 'roi': {0: 0.4449309386008759, 140: 0.11955965746663, 823: 0.06403981740598495, 1157: 0}, # noqa: E501 'stoploss': {'stoploss': -0.338070047333259}}, - 'results_metrics': {'trade_count': 1, 'avg_profit': 0.12357, 'total_profit': 6.185e-05, 'profit': 0.12357, 'duration': 1200.0}, # noqa: E501 + 'results_metrics': {'trade_count': 1, 'avg_profit': 0.12357, 'median_profit': -1.2222, 'total_profit': 6.185e-05, 'profit': 0.12357, 'duration': 1200.0}, # noqa: E501 'results_explanation': ' 1 trades. Avg profit 0.12%. Total profit 0.00006185 BTC ( 0.12Σ%). Avg duration 1200.0 min.', # noqa: E501 'total_profit': 6.185e-05, 'current_epoch': 2, @@ -1791,7 +1812,7 @@ def hyperopt_results(): 'loss': 14.241196856510731, 'params_dict': {'mfi-value': 25, 'fastd-value': 16, 'adx-value': 29, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 98, 'sell-fastd-value': 72, 'sell-adx-value': 51, 'sell-rsi-value': 82, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 889, 'roi_t2': 533, 'roi_t3': 263, 'roi_p1': 0.04759065393663096, 'roi_p2': 0.1488819964638463, 'roi_p3': 0.4102801822104605, 'stoploss': -0.05394588767607611}, # noqa: E501 'params_details': {'buy': {'mfi-value': 25, 'fastd-value': 16, 'adx-value': 29, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 98, 'sell-fastd-value': 72, 'sell-adx-value': 51, 'sell-rsi-value': 82, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.6067528326109377, 263: 0.19647265040047726, 796: 0.04759065393663096, 1685: 0}, 'stoploss': {'stoploss': -0.05394588767607611}}, # noqa: E501 - 'results_metrics': {'trade_count': 621, 'avg_profit': -0.43883302093397747, 'total_profit': -0.13639474, 'profit': -272.515306, 'duration': 1691.207729468599}, # noqa: E501 + 'results_metrics': {'trade_count': 621, 'avg_profit': -0.43883302093397747, 'median_profit': -1.2222, 'total_profit': -0.13639474, 'profit': -272.515306, 'duration': 1691.207729468599}, # noqa: E501 'results_explanation': ' 621 trades. Avg profit -0.44%. Total profit -0.13639474 BTC (-272.52Σ%). Avg duration 1691.2 min.', # noqa: E501 'total_profit': -0.13639474, 'current_epoch': 3, @@ -1801,14 +1822,14 @@ def hyperopt_results(): 'loss': 100000, 'params_dict': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1402, 'roi_t2': 676, 'roi_t3': 215, 'roi_p1': 0.06264755784937427, 'roi_p2': 0.14258587851894644, 'roi_p3': 0.20671291201040828, 'stoploss': -0.11818343570194478}, # noqa: E501 'params_details': {'buy': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.411946348378729, 215: 0.2052334363683207, 891: 0.06264755784937427, 2293: 0}, 'stoploss': {'stoploss': -0.11818343570194478}}, # noqa: E501 - 'results_metrics': {'trade_count': 0, 'avg_profit': None, 'total_profit': 0, 'profit': 0.0, 'duration': None}, # noqa: E501 + 'results_metrics': {'trade_count': 0, 'avg_profit': None, 'median_profit': None, 'total_profit': 0, 'profit': 0.0, 'duration': None}, # noqa: E501 'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501 'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_best': False }, { 'loss': 0.22195522184191518, 'params_dict': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 1269, 'roi_t2': 601, 'roi_t3': 444, 'roi_p1': 0.07280999507931168, 'roi_p2': 0.08946698095898986, 'roi_p3': 0.1454876733325284, 'stoploss': -0.18181041180901014}, # noqa: E501 'params_details': {'buy': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.3077646493708299, 444: 0.16227697603830155, 1045: 0.07280999507931168, 2314: 0}, 'stoploss': {'stoploss': -0.18181041180901014}}, # noqa: E501 - 'results_metrics': {'trade_count': 14, 'avg_profit': -0.3539515, 'total_profit': -0.002480140000000001, 'profit': -4.955321, 'duration': 3402.8571428571427}, # noqa: E501 + 'results_metrics': {'trade_count': 14, 'avg_profit': -0.3539515, 'median_profit': -1.2222, 'total_profit': -0.002480140000000001, 'profit': -4.955321, 'duration': 3402.8571428571427}, # noqa: E501 'results_explanation': ' 14 trades. Avg profit -0.35%. Total profit -0.00248014 BTC ( -4.96Σ%). Avg duration 3402.9 min.', # noqa: E501 'total_profit': -0.002480140000000001, 'current_epoch': 5, @@ -1818,7 +1839,7 @@ def hyperopt_results(): 'loss': 0.545315889154162, 'params_dict': {'mfi-value': 22, 'fastd-value': 43, 'adx-value': 46, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'bb_lower', 'sell-mfi-value': 87, 'sell-fastd-value': 65, 'sell-adx-value': 94, 'sell-rsi-value': 63, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 319, 'roi_t2': 556, 'roi_t3': 216, 'roi_p1': 0.06251955472249589, 'roi_p2': 0.11659519602202795, 'roi_p3': 0.0953744132197762, 'stoploss': -0.024551752215582423}, # noqa: E501 'params_details': {'buy': {'mfi-value': 22, 'fastd-value': 43, 'adx-value': 46, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 65, 'sell-adx-value': 94, 'sell-rsi-value': 63, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.2744891639643, 216: 0.17911475074452382, 772: 0.06251955472249589, 1091: 0}, 'stoploss': {'stoploss': -0.024551752215582423}}, # noqa: E501 - 'results_metrics': {'trade_count': 39, 'avg_profit': -0.21400679487179478, 'total_profit': -0.0041773, 'profit': -8.346264999999997, 'duration': 636.9230769230769}, # noqa: E501 + 'results_metrics': {'trade_count': 39, 'avg_profit': -0.21400679487179478, 'median_profit': -1.2222, 'total_profit': -0.0041773, 'profit': -8.346264999999997, 'duration': 636.9230769230769}, # noqa: E501 'results_explanation': ' 39 trades. Avg profit -0.21%. Total profit -0.00417730 BTC ( -8.35Σ%). Avg duration 636.9 min.', # noqa: E501 'total_profit': -0.0041773, 'current_epoch': 6, @@ -1830,7 +1851,7 @@ def hyperopt_results(): 'params_details': { 'buy': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.4837436938134452, 145: 0.10853310701097472, 765: 0.0586919200378493, 1536: 0}, # noqa: E501 'stoploss': {'stoploss': -0.14613268022709905}}, # noqa: E501 - 'results_metrics': {'trade_count': 318, 'avg_profit': -0.39833954716981146, 'total_profit': -0.06339929, 'profit': -126.67197600000004, 'duration': 3140.377358490566}, # noqa: E501 + 'results_metrics': {'trade_count': 318, 'avg_profit': -0.39833954716981146, 'median_profit': -1.2222, 'total_profit': -0.06339929, 'profit': -126.67197600000004, 'duration': 3140.377358490566}, # noqa: E501 'results_explanation': ' 318 trades. Avg profit -0.40%. Total profit -0.06339929 BTC (-126.67Σ%). Avg duration 3140.4 min.', # noqa: E501 'total_profit': -0.06339929, 'current_epoch': 7, @@ -1840,7 +1861,7 @@ def hyperopt_results(): 'loss': 20.0, # noqa: E501 'params_dict': {'mfi-value': 24, 'fastd-value': 43, 'adx-value': 33, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'sar_reversal', 'sell-mfi-value': 89, 'sell-fastd-value': 74, 'sell-adx-value': 70, 'sell-rsi-value': 70, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': False, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 1149, 'roi_t2': 375, 'roi_t3': 289, 'roi_p1': 0.05571820757172588, 'roi_p2': 0.0606240398618907, 'roi_p3': 0.1729012220156157, 'stoploss': -0.1588514289110401}, # noqa: E501 'params_details': {'buy': {'mfi-value': 24, 'fastd-value': 43, 'adx-value': 33, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 89, 'sell-fastd-value': 74, 'sell-adx-value': 70, 'sell-rsi-value': 70, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': False, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, 'roi': {0: 0.2892434694492323, 289: 0.11634224743361658, 664: 0.05571820757172588, 1813: 0}, 'stoploss': {'stoploss': -0.1588514289110401}}, # noqa: E501 - 'results_metrics': {'trade_count': 1, 'avg_profit': 0.0, 'total_profit': 0.0, 'profit': 0.0, 'duration': 5340.0}, # noqa: E501 + 'results_metrics': {'trade_count': 1, 'avg_profit': 0.0, 'median_profit': 0.0, 'total_profit': 0.0, 'profit': 0.0, 'duration': 5340.0}, # noqa: E501 'results_explanation': ' 1 trades. Avg profit 0.00%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration 5340.0 min.', # noqa: E501 'total_profit': 0.0, 'current_epoch': 8, @@ -1850,7 +1871,7 @@ def hyperopt_results(): 'loss': 2.4731817780991223, 'params_dict': {'mfi-value': 22, 'fastd-value': 20, 'adx-value': 29, 'rsi-value': 40, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'sar_reversal', 'sell-mfi-value': 97, 'sell-fastd-value': 65, 'sell-adx-value': 81, 'sell-rsi-value': 64, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1012, 'roi_t2': 584, 'roi_t3': 422, 'roi_p1': 0.036764323603472565, 'roi_p2': 0.10335480573205287, 'roi_p3': 0.10322347377503042, 'stoploss': -0.2780610808108503}, # noqa: E501 'params_details': {'buy': {'mfi-value': 22, 'fastd-value': 20, 'adx-value': 29, 'rsi-value': 40, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 97, 'sell-fastd-value': 65, 'sell-adx-value': 81, 'sell-rsi-value': 64, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.2433426031105559, 422: 0.14011912933552545, 1006: 0.036764323603472565, 2018: 0}, 'stoploss': {'stoploss': -0.2780610808108503}}, # noqa: E501 - 'results_metrics': {'trade_count': 229, 'avg_profit': -0.38433433624454144, 'total_profit': -0.044050070000000004, 'profit': -88.01256299999999, 'duration': 6505.676855895196}, # noqa: E501 + 'results_metrics': {'trade_count': 229, 'avg_profit': -0.38433433624454144, 'median_profit': -1.2222, 'total_profit': -0.044050070000000004, 'profit': -88.01256299999999, 'duration': 6505.676855895196}, # noqa: E501 'results_explanation': ' 229 trades. Avg profit -0.38%. Total profit -0.04405007 BTC ( -88.01Σ%). Avg duration 6505.7 min.', # noqa: E501 'total_profit': -0.044050070000000004, # noqa: E501 'current_epoch': 9, @@ -1860,7 +1881,7 @@ def hyperopt_results(): 'loss': -0.2604606005845212, # noqa: E501 'params_dict': {'mfi-value': 23, 'fastd-value': 24, 'adx-value': 22, 'rsi-value': 24, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 97, 'sell-fastd-value': 70, 'sell-adx-value': 64, 'sell-rsi-value': 80, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 792, 'roi_t2': 464, 'roi_t3': 215, 'roi_p1': 0.04594053535385903, 'roi_p2': 0.09623192684243963, 'roi_p3': 0.04428219070850663, 'stoploss': -0.16992287161634415}, # noqa: E501 'params_details': {'buy': {'mfi-value': 23, 'fastd-value': 24, 'adx-value': 22, 'rsi-value': 24, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 97, 'sell-fastd-value': 70, 'sell-adx-value': 64, 'sell-rsi-value': 80, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, 'roi': {0: 0.18645465290480528, 215: 0.14217246219629864, 679: 0.04594053535385903, 1471: 0}, 'stoploss': {'stoploss': -0.16992287161634415}}, # noqa: E501 - 'results_metrics': {'trade_count': 4, 'avg_profit': 0.1080385, 'total_profit': 0.00021629, 'profit': 0.432154, 'duration': 2850.0}, # noqa: E501 + 'results_metrics': {'trade_count': 4, 'avg_profit': 0.1080385, 'median_profit': -1.2222, 'total_profit': 0.00021629, 'profit': 0.432154, 'duration': 2850.0}, # noqa: E501 'results_explanation': ' 4 trades. Avg profit 0.11%. Total profit 0.00021629 BTC ( 0.43Σ%). Avg duration 2850.0 min.', # noqa: E501 'total_profit': 0.00021629, 'current_epoch': 10, @@ -1870,7 +1891,7 @@ def hyperopt_results(): 'loss': 4.876465945994304, # noqa: E501 'params_dict': {'mfi-value': 20, 'fastd-value': 32, 'adx-value': 49, 'rsi-value': 23, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower', 'sell-mfi-value': 75, 'sell-fastd-value': 56, 'sell-adx-value': 61, 'sell-rsi-value': 62, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 579, 'roi_t2': 614, 'roi_t3': 273, 'roi_p1': 0.05307643172744114, 'roi_p2': 0.1352282078262871, 'roi_p3': 0.1913307406325751, 'stoploss': -0.25728526022513887}, # noqa: E501 'params_details': {'buy': {'mfi-value': 20, 'fastd-value': 32, 'adx-value': 49, 'rsi-value': 23, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 75, 'sell-fastd-value': 56, 'sell-adx-value': 61, 'sell-rsi-value': 62, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.3796353801863034, 273: 0.18830463955372825, 887: 0.05307643172744114, 1466: 0}, 'stoploss': {'stoploss': -0.25728526022513887}}, # noqa: E501 - 'results_metrics': {'trade_count': 117, 'avg_profit': -1.2698609145299145, 'total_profit': -0.07436117, 'profit': -148.573727, 'duration': 4282.5641025641025}, # noqa: E501 + 'results_metrics': {'trade_count': 117, 'avg_profit': -1.2698609145299145, 'median_profit': -1.2222, 'total_profit': -0.07436117, 'profit': -148.573727, 'duration': 4282.5641025641025}, # noqa: E501 'results_explanation': ' 117 trades. Avg profit -1.27%. Total profit -0.07436117 BTC (-148.57Σ%). Avg duration 4282.6 min.', # noqa: E501 'total_profit': -0.07436117, 'current_epoch': 11, @@ -1880,7 +1901,7 @@ def hyperopt_results(): 'loss': 100000, 'params_dict': {'mfi-value': 10, 'fastd-value': 36, 'adx-value': 31, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'sar_reversal', 'sell-mfi-value': 80, 'sell-fastd-value': 71, 'sell-adx-value': 60, 'sell-rsi-value': 85, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1156, 'roi_t2': 581, 'roi_t3': 408, 'roi_p1': 0.06860454019988212, 'roi_p2': 0.12473718444931989, 'roi_p3': 0.2896360635226823, 'stoploss': -0.30889015124682806}, # noqa: E501 'params_details': {'buy': {'mfi-value': 10, 'fastd-value': 36, 'adx-value': 31, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 80, 'sell-fastd-value': 71, 'sell-adx-value': 60, 'sell-rsi-value': 85, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4829777881718843, 408: 0.19334172464920202, 989: 0.06860454019988212, 2145: 0}, 'stoploss': {'stoploss': -0.30889015124682806}}, # noqa: E501 - 'results_metrics': {'trade_count': 0, 'avg_profit': None, 'total_profit': 0, 'profit': 0.0, 'duration': None}, # noqa: E501 + 'results_metrics': {'trade_count': 0, 'avg_profit': None, 'median_profit': None, 'total_profit': 0, 'profit': 0.0, 'duration': None}, # noqa: E501 'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501 'total_profit': 0, 'current_epoch': 12, @@ -1888,3 +1909,142 @@ def hyperopt_results(): 'is_best': False } ] + + +@pytest.fixture +def saved_hyperopt_results(): + hyperopt_res = [ + { + 'loss': 0.4366182531161519, + 'params_dict': { + 'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1190, 'roi_t2': 541, 'roi_t3': 408, 'roi_p1': 0.026035863879169705, 'roi_p2': 0.12508730043628782, 'roi_p3': 0.27766427921605896, 'stoploss': -0.2562930402099556}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4287874435315165, 408: 0.15112316431545753, 949: 0.026035863879169705, 2139: 0}, 'stoploss': {'stoploss': -0.2562930402099556}}, # noqa: E501 + 'results_metrics': {'total_trades': 2, 'wins': 0, 'draws': 0, 'losses': 2, 'profit_mean': -0.01254995, 'profit_median': -0.012222, 'profit_total': -0.00125625, 'profit_total_abs': -2.50999, 'holding_avg': timedelta(minutes=3930.0)}, # noqa: E501 + 'results_explanation': ' 2 trades. Avg profit -1.25%. Total profit -0.00125625 BTC ( -2.51Σ%). Avg duration 3930.0 min.', # noqa: E501 + 'total_profit': -0.00125625, + 'current_epoch': 1, + 'is_initial_point': True, + 'is_best': True + }, { + 'loss': 20.0, + 'params_dict': { + 'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 334, 'roi_t2': 683, 'roi_t3': 140, 'roi_p1': 0.06403981740598495, 'roi_p2': 0.055519840060645045, 'roi_p3': 0.3253712811342459, 'stoploss': -0.338070047333259}, # noqa: E501 + 'params_details': { + 'buy': {'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, # noqa: E501 + 'sell': {'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, # noqa: E501 + 'roi': {0: 0.4449309386008759, 140: 0.11955965746663, 823: 0.06403981740598495, 1157: 0}, # noqa: E501 + 'stoploss': {'stoploss': -0.338070047333259}}, + 'results_metrics': {'total_trades': 1, 'wins': 0, 'draws': 0, 'losses': 1, 'profit_mean': 0.012357, 'profit_median': -0.012222, 'profit_total': 6.185e-05, 'profit_total_abs': 0.12357, 'holding_avg': timedelta(minutes=1200.0)}, # noqa: E501 + 'results_explanation': ' 1 trades. Avg profit 0.12%. Total profit 0.00006185 BTC ( 0.12Σ%). Avg duration 1200.0 min.', # noqa: E501 + 'total_profit': 6.185e-05, + 'current_epoch': 2, + 'is_initial_point': True, + 'is_best': False + }, { + 'loss': 14.241196856510731, + 'params_dict': {'mfi-value': 25, 'fastd-value': 16, 'adx-value': 29, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 98, 'sell-fastd-value': 72, 'sell-adx-value': 51, 'sell-rsi-value': 82, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 889, 'roi_t2': 533, 'roi_t3': 263, 'roi_p1': 0.04759065393663096, 'roi_p2': 0.1488819964638463, 'roi_p3': 0.4102801822104605, 'stoploss': -0.05394588767607611}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 25, 'fastd-value': 16, 'adx-value': 29, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 98, 'sell-fastd-value': 72, 'sell-adx-value': 51, 'sell-rsi-value': 82, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.6067528326109377, 263: 0.19647265040047726, 796: 0.04759065393663096, 1685: 0}, 'stoploss': {'stoploss': -0.05394588767607611}}, # noqa: E501 + 'results_metrics': {'total_trades': 621, 'wins': 320, 'draws': 0, 'losses': 301, 'profit_mean': -0.043883302093397747, 'profit_median': -0.012222, 'profit_total': -0.13639474, 'profit_total_abs': -272.515306, 'holding_avg': timedelta(minutes=1691.207729468599)}, # noqa: E501 + 'results_explanation': ' 621 trades. Avg profit -0.44%. Total profit -0.13639474 BTC (-272.52Σ%). Avg duration 1691.2 min.', # noqa: E501 + 'total_profit': -0.13639474, + 'current_epoch': 3, + 'is_initial_point': True, + 'is_best': False + }, { + 'loss': 100000, + 'params_dict': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1402, 'roi_t2': 676, 'roi_t3': 215, 'roi_p1': 0.06264755784937427, 'roi_p2': 0.14258587851894644, 'roi_p3': 0.20671291201040828, 'stoploss': -0.11818343570194478}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 13, 'fastd-value': 35, 'adx-value': 39, 'rsi-value': 29, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 54, 'sell-adx-value': 63, 'sell-rsi-value': 93, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.411946348378729, 215: 0.2052334363683207, 891: 0.06264755784937427, 2293: 0}, 'stoploss': {'stoploss': -0.11818343570194478}}, # noqa: E501 + 'results_metrics': {'total_trades': 0, 'wins': 0, 'draws': 0, 'losses': 0, 'profit_mean': None, 'profit_median': None, 'profit_total': 0, 'profit': 0.0, 'holding_avg': timedelta()}, # noqa: E501 + 'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501 + 'total_profit': 0, 'current_epoch': 4, 'is_initial_point': True, 'is_best': False + }, { + 'loss': 0.22195522184191518, + 'params_dict': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 1269, 'roi_t2': 601, 'roi_t3': 444, 'roi_p1': 0.07280999507931168, 'roi_p2': 0.08946698095898986, 'roi_p3': 0.1454876733325284, 'stoploss': -0.18181041180901014}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 17, 'fastd-value': 21, 'adx-value': 38, 'rsi-value': 33, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 82, 'sell-adx-value': 78, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.3077646493708299, 444: 0.16227697603830155, 1045: 0.07280999507931168, 2314: 0}, 'stoploss': {'stoploss': -0.18181041180901014}}, # noqa: E501 + 'results_metrics': {'total_trades': 14, 'wins': 6, 'draws': 0, 'losses': 8, 'profit_mean': -0.003539515, 'profit_median': -0.012222, 'profit_total': -0.002480140000000001, 'profit_total_abs': -4.955321, 'holding_avg': timedelta(minutes=3402.8571428571427)}, # noqa: E501 + 'results_explanation': ' 14 trades. Avg profit -0.35%. Total profit -0.00248014 BTC ( -4.96Σ%). Avg duration 3402.9 min.', # noqa: E501 + 'total_profit': -0.002480140000000001, + 'current_epoch': 5, + 'is_initial_point': True, + 'is_best': True + }, { + 'loss': 0.545315889154162, + 'params_dict': {'mfi-value': 22, 'fastd-value': 43, 'adx-value': 46, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'bb_lower', 'sell-mfi-value': 87, 'sell-fastd-value': 65, 'sell-adx-value': 94, 'sell-rsi-value': 63, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 319, 'roi_t2': 556, 'roi_t3': 216, 'roi_p1': 0.06251955472249589, 'roi_p2': 0.11659519602202795, 'roi_p3': 0.0953744132197762, 'stoploss': -0.024551752215582423}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 22, 'fastd-value': 43, 'adx-value': 46, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 87, 'sell-fastd-value': 65, 'sell-adx-value': 94, 'sell-rsi-value': 63, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.2744891639643, 216: 0.17911475074452382, 772: 0.06251955472249589, 1091: 0}, 'stoploss': {'stoploss': -0.024551752215582423}}, # noqa: E501 + 'results_metrics': {'total_trades': 39, 'wins': 20, 'draws': 0, 'losses': 19, 'profit_mean': -0.0021400679487179478, 'profit_median': -0.012222, 'profit_total': -0.0041773, 'profit_total_abs': -8.346264999999997, 'holding_avg': timedelta(minutes=636.9230769230769)}, # noqa: E501 + 'results_explanation': ' 39 trades. Avg profit -0.21%. Total profit -0.00417730 BTC ( -8.35Σ%). Avg duration 636.9 min.', # noqa: E501 + 'total_profit': -0.0041773, + 'current_epoch': 6, + 'is_initial_point': True, + 'is_best': False + }, { + 'loss': 4.713497421432944, + 'params_dict': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower', 'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 771, 'roi_t2': 620, 'roi_t3': 145, 'roi_p1': 0.0586919200378493, 'roi_p2': 0.04984118697312542, 'roi_p3': 0.37521058680247044, 'stoploss': -0.14613268022709905}, # noqa: E501 + 'params_details': { + 'buy': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.4837436938134452, 145: 0.10853310701097472, 765: 0.0586919200378493, 1536: 0}, # noqa: E501 + 'stoploss': {'stoploss': -0.14613268022709905}}, # noqa: E501 + 'results_metrics': {'total_trades': 318, 'wins': 100, 'draws': 0, 'losses': 218, 'profit_mean': -0.0039833954716981146, 'profit_median': -0.012222, 'profit_total': -0.06339929, 'profit_total_abs': -126.67197600000004, 'holding_avg': timedelta(minutes=3140.377358490566)}, # noqa: E501 + 'results_explanation': ' 318 trades. Avg profit -0.40%. Total profit -0.06339929 BTC (-126.67Σ%). Avg duration 3140.4 min.', # noqa: E501 + 'total_profit': -0.06339929, + 'current_epoch': 7, + 'is_initial_point': True, + 'is_best': False + }, { + 'loss': 20.0, # noqa: E501 + 'params_dict': {'mfi-value': 24, 'fastd-value': 43, 'adx-value': 33, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'sar_reversal', 'sell-mfi-value': 89, 'sell-fastd-value': 74, 'sell-adx-value': 70, 'sell-rsi-value': 70, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': False, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 1149, 'roi_t2': 375, 'roi_t3': 289, 'roi_p1': 0.05571820757172588, 'roi_p2': 0.0606240398618907, 'roi_p3': 0.1729012220156157, 'stoploss': -0.1588514289110401}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 24, 'fastd-value': 43, 'adx-value': 33, 'rsi-value': 20, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 89, 'sell-fastd-value': 74, 'sell-adx-value': 70, 'sell-rsi-value': 70, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': False, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, 'roi': {0: 0.2892434694492323, 289: 0.11634224743361658, 664: 0.05571820757172588, 1813: 0}, 'stoploss': {'stoploss': -0.1588514289110401}}, # noqa: E501 + 'results_metrics': {'total_trades': 1, 'wins': 0, 'draws': 1, 'losses': 0, 'profit_mean': 0.0, 'profit_median': 0.0, 'profit_total': 0.0, 'profit_total_abs': 0.0, 'holding_avg': timedelta(minutes=5340.0)}, # noqa: E501 + 'results_explanation': ' 1 trades. Avg profit 0.00%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration 5340.0 min.', # noqa: E501 + 'total_profit': 0.0, + 'current_epoch': 8, + 'is_initial_point': True, + 'is_best': False + }, { + 'loss': 2.4731817780991223, + 'params_dict': {'mfi-value': 22, 'fastd-value': 20, 'adx-value': 29, 'rsi-value': 40, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'sar_reversal', 'sell-mfi-value': 97, 'sell-fastd-value': 65, 'sell-adx-value': 81, 'sell-rsi-value': 64, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1012, 'roi_t2': 584, 'roi_t3': 422, 'roi_p1': 0.036764323603472565, 'roi_p2': 0.10335480573205287, 'roi_p3': 0.10322347377503042, 'stoploss': -0.2780610808108503}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 22, 'fastd-value': 20, 'adx-value': 29, 'rsi-value': 40, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 97, 'sell-fastd-value': 65, 'sell-adx-value': 81, 'sell-rsi-value': 64, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.2433426031105559, 422: 0.14011912933552545, 1006: 0.036764323603472565, 2018: 0}, 'stoploss': {'stoploss': -0.2780610808108503}}, # noqa: E501 + 'results_metrics': {'total_trades': 229, 'wins': 150, 'draws': 0, 'losses': 79, 'profit_mean': -0.0038433433624454144, 'profit_median': -0.012222, 'profit_total': -0.044050070000000004, 'profit_total_abs': -88.01256299999999, 'holding_avg': timedelta(minutes=6505.676855895196)}, # noqa: E501 + 'results_explanation': ' 229 trades. Avg profit -0.38%. Total profit -0.04405007 BTC ( -88.01Σ%). Avg duration 6505.7 min.', # noqa: E501 + 'total_profit': -0.044050070000000004, # noqa: E501 + 'current_epoch': 9, + 'is_initial_point': True, + 'is_best': False + }, { + 'loss': -0.2604606005845212, # noqa: E501 + 'params_dict': {'mfi-value': 23, 'fastd-value': 24, 'adx-value': 22, 'rsi-value': 24, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 97, 'sell-fastd-value': 70, 'sell-adx-value': 64, 'sell-rsi-value': 80, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 792, 'roi_t2': 464, 'roi_t3': 215, 'roi_p1': 0.04594053535385903, 'roi_p2': 0.09623192684243963, 'roi_p3': 0.04428219070850663, 'stoploss': -0.16992287161634415}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 23, 'fastd-value': 24, 'adx-value': 22, 'rsi-value': 24, 'mfi-enabled': False, 'fastd-enabled': False, 'adx-enabled': False, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 97, 'sell-fastd-value': 70, 'sell-adx-value': 64, 'sell-rsi-value': 80, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, 'roi': {0: 0.18645465290480528, 215: 0.14217246219629864, 679: 0.04594053535385903, 1471: 0}, 'stoploss': {'stoploss': -0.16992287161634415}}, # noqa: E501 + 'results_metrics': {'total_trades': 4, 'wins': 0, 'draws': 0, 'losses': 4, 'profit_mean': 0.001080385, 'profit_median': -0.012222, 'profit_total': 0.00021629, 'profit_total_abs': 0.432154, 'holding_avg': timedelta(minutes=2850.0)}, # noqa: E501 + 'results_explanation': ' 4 trades. Avg profit 0.11%. Total profit 0.00021629 BTC ( 0.43Σ%). Avg duration 2850.0 min.', # noqa: E501 + 'total_profit': 0.00021629, + 'current_epoch': 10, + 'is_initial_point': True, + 'is_best': True + }, { + 'loss': 4.876465945994304, # noqa: E501 + 'params_dict': {'mfi-value': 20, 'fastd-value': 32, 'adx-value': 49, 'rsi-value': 23, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower', 'sell-mfi-value': 75, 'sell-fastd-value': 56, 'sell-adx-value': 61, 'sell-rsi-value': 62, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 579, 'roi_t2': 614, 'roi_t3': 273, 'roi_p1': 0.05307643172744114, 'roi_p2': 0.1352282078262871, 'roi_p3': 0.1913307406325751, 'stoploss': -0.25728526022513887}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 20, 'fastd-value': 32, 'adx-value': 49, 'rsi-value': 23, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 75, 'sell-fastd-value': 56, 'sell-adx-value': 61, 'sell-rsi-value': 62, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.3796353801863034, 273: 0.18830463955372825, 887: 0.05307643172744114, 1466: 0}, 'stoploss': {'stoploss': -0.25728526022513887}}, # noqa: E501 + # New Hyperopt mode! + 'results_metrics': {'total_trades': 117, 'wins': 67, 'draws': 0, 'losses': 50, 'profit_mean': -0.012698609145299145, 'profit_median': -0.012222, 'profit_total': -0.07436117, 'profit_total_abs': -148.573727, 'holding_avg': timedelta(minutes=4282.5641025641025)}, # noqa: E501 + 'results_explanation': ' 117 trades. Avg profit -1.27%. Total profit -0.07436117 BTC (-148.57Σ%). Avg duration 4282.6 min.', # noqa: E501 + 'total_profit': -0.07436117, + 'current_epoch': 11, + 'is_initial_point': True, + 'is_best': False + }, { + 'loss': 100000, + 'params_dict': {'mfi-value': 10, 'fastd-value': 36, 'adx-value': 31, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'sar_reversal', 'sell-mfi-value': 80, 'sell-fastd-value': 71, 'sell-adx-value': 60, 'sell-rsi-value': 85, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1156, 'roi_t2': 581, 'roi_t3': 408, 'roi_p1': 0.06860454019988212, 'roi_p2': 0.12473718444931989, 'roi_p3': 0.2896360635226823, 'stoploss': -0.30889015124682806}, # noqa: E501 + 'params_details': {'buy': {'mfi-value': 10, 'fastd-value': 36, 'adx-value': 31, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': False, 'trigger': 'sar_reversal'}, 'sell': {'sell-mfi-value': 80, 'sell-fastd-value': 71, 'sell-adx-value': 60, 'sell-rsi-value': 85, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4829777881718843, 408: 0.19334172464920202, 989: 0.06860454019988212, 2145: 0}, 'stoploss': {'stoploss': -0.30889015124682806}}, # noqa: E501 + 'results_metrics': {'total_trades': 0, 'wins': 0, 'draws': 0, 'losses': 0, 'profit_mean': None, 'profit_median': None, 'profit_total': 0, 'profit_total_abs': 0.0, 'holding_avg': timedelta()}, # noqa: E501 + 'results_explanation': ' 0 trades. Avg profit nan%. Total profit 0.00000000 BTC ( 0.00Σ%). Avg duration nan min.', # noqa: E501 + 'total_profit': 0, + 'current_epoch': 12, + 'is_initial_point': True, + 'is_best': False + } + ] + + for res in hyperopt_res: + res['results_metrics']['holding_avg_s'] = res['results_metrics']['holding_avg' + ].total_seconds() + + return hyperopt_res diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index fa9910b8d..b92b51144 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -28,8 +28,10 @@ def mock_trade_1(fee): amount_requested=123.0, fee_open=fee.return_value, fee_close=fee.return_value, + is_open=True, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=17), open_rate=0.123, - exchange='bittrex', + exchange='binance', open_order_id='dry_run_buy_12345', strategy='DefaultStrategy', timeframe=5, @@ -81,14 +83,15 @@ def mock_trade_2(fee): open_rate=0.123, close_rate=0.128, close_profit=0.005, - exchange='bittrex', + close_profit_abs=0.000584127, + exchange='binance', is_open=False, open_order_id='dry_run_sell_12345', strategy='DefaultStrategy', timeframe=5, sell_reason='sell_signal', open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), - close_date=datetime.now(tz=timezone.utc), + close_date=datetime.now(tz=timezone.utc) - timedelta(minutes=2), ) o = Order.parse_from_ccxt_object(mock_order_2(), 'ETC/BTC', 'buy') trade.orders.append(o) @@ -140,7 +143,8 @@ def mock_trade_3(fee): open_rate=0.05, close_rate=0.06, close_profit=0.01, - exchange='bittrex', + close_profit_abs=0.000155, + exchange='binance', is_open=False, strategy='DefaultStrategy', timeframe=5, @@ -180,8 +184,10 @@ def mock_trade_4(fee): amount_requested=124.0, fee_open=fee.return_value, fee_close=fee.return_value, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=14), + is_open=True, open_rate=0.123, - exchange='bittrex', + exchange='binance', open_order_id='prod_buy_12345', strategy='DefaultStrategy', timeframe=5, @@ -230,10 +236,13 @@ def mock_trade_5(fee): amount_requested=124.0, fee_open=fee.return_value, fee_close=fee.return_value, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=12), + is_open=True, open_rate=0.123, - exchange='bittrex', + exchange='binance', strategy='SampleStrategy', - stoploss_order_id='prod_stoploss_3455' + stoploss_order_id='prod_stoploss_3455', + timeframe=5, ) o = Order.parse_from_ccxt_object(mock_order_5(), 'XRP/BTC', 'buy') trade.orders.append(o) @@ -279,12 +288,15 @@ def mock_trade_6(fee): stake_amount=0.001, amount=2.0, amount_requested=2.0, + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=5), fee_open=fee.return_value, fee_close=fee.return_value, + is_open=True, open_rate=0.15, - exchange='bittrex', + exchange='binance', strategy='SampleStrategy', open_order_id="prod_sell_6", + timeframe=5, ) o = Order.parse_from_ccxt_object(mock_order_6(), 'LTC/BTC', 'buy') trade.orders.append(o) diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 96ac6f63c..6bde60926 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -1,3 +1,4 @@ +from math import isclose from pathlib import Path from unittest.mock import MagicMock @@ -8,11 +9,12 @@ from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, BT_DATA_COLUMNS_MID, BT_DATA_COLUMNS_OLD, - analyze_trade_parallelism, calculate_market_change, - calculate_max_drawdown, combine_dataframes_with_mean, - create_cum_profit, extract_trades_of_period, - get_latest_backtest_filename, get_latest_hyperopt_file, - load_backtest_data, load_trades, load_trades_from_db) + analyze_trade_parallelism, calculate_csum, + calculate_market_change, calculate_max_drawdown, + combine_dataframes_with_mean, create_cum_profit, + extract_trades_of_period, get_latest_backtest_filename, + get_latest_hyperopt_file, load_backtest_data, load_trades, + load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history from tests.conftest import create_mock_trades from tests.conftest_trades import MOCK_TRADE_COUNT @@ -245,7 +247,7 @@ def test_create_cum_profit(testdatadir): "cum_profits", timeframe="5m") assert "cum_profits" in cum_profits.columns assert cum_profits.iloc[0]['cum_profits'] == 0 - assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 + assert isclose(cum_profits.iloc[-1]['cum_profits'], 8.723007518796964e-06) def test_create_cum_profit1(testdatadir): @@ -263,7 +265,7 @@ def test_create_cum_profit1(testdatadir): "cum_profits", timeframe="5m") assert "cum_profits" in cum_profits.columns assert cum_profits.iloc[0]['cum_profits'] == 0 - assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 + assert isclose(cum_profits.iloc[-1]['cum_profits'], 8.723007518796964e-06) with pytest.raises(ValueError, match='Trade dataframe empty.'): create_cum_profit(df.set_index('date'), bt_data[bt_data["pair"] == 'NOTAPAIR'], @@ -273,15 +275,35 @@ def test_create_cum_profit1(testdatadir): def test_calculate_max_drawdown(testdatadir): filename = testdatadir / "backtest-result_test.json" bt_data = load_backtest_data(filename) - drawdown, h, low = calculate_max_drawdown(bt_data) + drawdown, hdate, lowdate, hval, lval = calculate_max_drawdown(bt_data) assert isinstance(drawdown, float) assert pytest.approx(drawdown) == 0.21142322 - assert isinstance(h, Timestamp) - assert isinstance(low, Timestamp) - assert h == Timestamp('2018-01-24 14:25:00', tz='UTC') - assert low == Timestamp('2018-01-30 04:45:00', tz='UTC') + assert isinstance(hdate, Timestamp) + assert isinstance(lowdate, Timestamp) + assert isinstance(hval, float) + assert isinstance(lval, float) + assert hdate == Timestamp('2018-01-24 14:25:00', tz='UTC') + assert lowdate == Timestamp('2018-01-30 04:45:00', tz='UTC') with pytest.raises(ValueError, match='Trade dataframe empty.'): - drawdown, h, low = calculate_max_drawdown(DataFrame()) + drawdown, hdate, lowdate, hval, lval = calculate_max_drawdown(DataFrame()) + + +def test_calculate_csum(testdatadir): + filename = testdatadir / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + csum_min, csum_max = calculate_csum(bt_data) + + assert isinstance(csum_min, float) + assert isinstance(csum_max, float) + assert csum_min < 0.01 + assert csum_max > 0.02 + csum_min1, csum_max1 = calculate_csum(bt_data, 5) + + assert csum_min1 == csum_min + 5 + assert csum_max1 == csum_max + 5 + + with pytest.raises(ValueError, match='Trade dataframe empty.'): + csum_min, csum_max = calculate_csum(DataFrame()) def test_calculate_max_drawdown2(): @@ -295,13 +317,16 @@ def test_calculate_max_drawdown2(): # sort by profit and reset index df = df.sort_values('profit').reset_index(drop=True) df1 = df.copy() - drawdown, h, low = calculate_max_drawdown(df, date_col='open_date', value_col='profit') + drawdown, hdate, ldate, hval, lval = calculate_max_drawdown( + df, date_col='open_date', value_col='profit') # Ensure df has not been altered. assert df.equals(df1) assert isinstance(drawdown, float) # High must be before low - assert h < low + assert hdate < ldate + # High value must be higher than low value + assert hval > lval assert drawdown == 0.091755 df = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_date']) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 4fdcce4d2..802fd4b12 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -1,5 +1,7 @@ # pragma pylint: disable=missing-docstring, C0103 import logging +from pathlib import Path +from shutil import copyfile import pytest @@ -10,8 +12,8 @@ from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_forma trades_to_ohlcv, trim_dataframe) from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) -from tests.conftest import log_has -from tests.data.test_history import _backup_file, _clean_test_file +from tests.conftest import log_has, log_has_re +from tests.data.test_history import _clean_test_file def test_dataframe_correct_columns(result): @@ -62,8 +64,8 @@ def test_ohlcv_fill_up_missing_data(testdatadir, caplog): # Column names should not change assert (data.columns == data2.columns).all() - assert log_has(f"Missing data fillup for UNITTEST/BTC: before: " - f"{len(data)} - after: {len(data2)}", caplog) + assert log_has_re(f"Missing data fillup for UNITTEST/BTC: before: " + f"{len(data)} - after: {len(data2)}.*", caplog) # Test fillup actually fixes invalid backtest data min_date, max_date = get_timerange({'UNITTEST/BTC': data}) @@ -125,8 +127,8 @@ def test_ohlcv_fill_up_missing_data2(caplog): # Column names should not change assert (data.columns == data2.columns).all() - assert log_has(f"Missing data fillup for UNITTEST/BTC: before: " - f"{len(data)} - after: {len(data2)}", caplog) + assert log_has_re(f"Missing data fillup for UNITTEST/BTC: before: " + f"{len(data)} - after: {len(data2)}.*", caplog) def test_ohlcv_drop_incomplete(caplog): @@ -197,6 +199,16 @@ def test_trim_dataframe(testdatadir) -> None: assert all(data_modify.iloc[-1] == data.iloc[-1]) assert all(data_modify.iloc[0] == data.iloc[30]) + data_modify = data.copy() + tr = TimeRange('date', None, min_date + 1800, 0) + # Remove first 20 candles - ignores min date + data_modify = trim_dataframe(data_modify, tr, startup_candles=20) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 20 + assert all(data_modify.iloc[-1] == data.iloc[-1]) + assert all(data_modify.iloc[0] == data.iloc[20]) + data_modify = data.copy() # Remove last 30 minutes (1800 s) tr = TimeRange(None, 'date', 0, max_date - 1800) @@ -241,17 +253,18 @@ def test_trades_dict_to_list(fetch_trades_result): assert t[6] == fetch_trades_result[i]['cost'] -def test_convert_trades_format(mocker, default_conf, testdatadir): - files = [{'old': testdatadir / "XRP_ETH-trades.json.gz", - 'new': testdatadir / "XRP_ETH-trades.json"}, - {'old': testdatadir / "XRP_OLD-trades.json.gz", - 'new': testdatadir / "XRP_OLD-trades.json"}, +def test_convert_trades_format(default_conf, testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) + files = [{'old': tmpdir1 / "XRP_ETH-trades.json.gz", + 'new': tmpdir1 / "XRP_ETH-trades.json"}, + {'old': tmpdir1 / "XRP_OLD-trades.json.gz", + 'new': tmpdir1 / "XRP_OLD-trades.json"}, ] for file in files: - _backup_file(file['old'], copy_file=True) + copyfile(testdatadir / file['old'].name, file['old']) assert not file['new'].exists() - default_conf['datadir'] = testdatadir + default_conf['datadir'] = tmpdir1 convert_trades_format(default_conf, convert_from='jsongz', convert_to='json', erase=False) @@ -274,14 +287,20 @@ def test_convert_trades_format(mocker, default_conf, testdatadir): file['new'].unlink() -def test_convert_ohlcv_format(mocker, default_conf, testdatadir): - file1 = testdatadir / "XRP_ETH-5m.json" - file1_new = testdatadir / "XRP_ETH-5m.json.gz" - file2 = testdatadir / "XRP_ETH-1m.json" - file2_new = testdatadir / "XRP_ETH-1m.json.gz" - _backup_file(file1, copy_file=True) - _backup_file(file2, copy_file=True) - default_conf['datadir'] = testdatadir +def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) + + file1_orig = testdatadir / "XRP_ETH-5m.json" + file1 = tmpdir1 / "XRP_ETH-5m.json" + file1_new = tmpdir1 / "XRP_ETH-5m.json.gz" + file2_orig = testdatadir / "XRP_ETH-1m.json" + file2 = tmpdir1 / "XRP_ETH-1m.json" + file2_new = tmpdir1 / "XRP_ETH-1m.json.gz" + + copyfile(file1_orig, file1) + copyfile(file2_orig, file2) + + default_conf['datadir'] = tmpdir1 default_conf['pairs'] = ['XRP_ETH'] default_conf['timeframes'] = ['1m', '5m'] @@ -307,10 +326,3 @@ def test_convert_ohlcv_format(mocker, default_conf, testdatadir): assert file2.exists() assert not file1_new.exists() assert not file2_new.exists() - - _clean_test_file(file1) - _clean_test_file(file2) - if file1_new.exists(): - file1_new.unlink() - if file2_new.exists(): - file2_new.unlink() diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index ee2e551b6..e43309743 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -5,9 +5,9 @@ import pytest from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.plugins.pairlistmanager import PairListManager -from freqtrade.state import RunMode from tests.conftest import get_patched_exchange @@ -214,8 +214,8 @@ def test_current_whitelist(mocker, default_conf, tickers): pairlist.refresh_pairlist() assert dp.current_whitelist() == pairlist._whitelist - # The identity of the 2 lists should be identical - assert dp.current_whitelist() is pairlist._whitelist + # The identity of the 2 lists should not be identical, but a copy + assert dp.current_whitelist() is not pairlist._whitelist with pytest.raises(OperationalException): dp = DataProvider(default_conf, exchange) @@ -246,3 +246,46 @@ def test_get_analyzed_dataframe(mocker, default_conf, ohlcv_history): assert dataframe.empty assert isinstance(time, datetime) assert time == datetime(1970, 1, 1, tzinfo=timezone.utc) + + # Test backtest mode + default_conf["runmode"] = RunMode.BACKTEST + dp._set_dataframe_max_index(1) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + + assert len(dataframe) == 1 + + dp._set_dataframe_max_index(2) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert len(dataframe) == 2 + + dp._set_dataframe_max_index(3) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert len(dataframe) == 3 + + dp._set_dataframe_max_index(500) + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert len(dataframe) == len(ohlcv_history) + + +def test_no_exchange_mode(default_conf): + dp = DataProvider(default_conf, None) + + message = "Exchange is not available to DataProvider." + + with pytest.raises(OperationalException, match=message): + dp.refresh([()]) + + with pytest.raises(OperationalException, match=message): + dp.ohlcv('XRP/USDT', '5m') + + with pytest.raises(OperationalException, match=message): + dp.market('XRP/USDT') + + with pytest.raises(OperationalException, match=message): + dp.ticker('XRP/USDT') + + with pytest.raises(OperationalException, match=message): + dp.orderbook('XRP/USDT', 20) + + with pytest.raises(OperationalException, match=message): + dp.available_pairs() diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 353cfc6f7..d203d0792 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -86,14 +86,12 @@ def test_load_data_7min_timeframe(mocker, caplog, default_conf, testdatadir) -> def test_load_data_1min_timeframe(ohlcv_history, mocker, caplog, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history) file = testdatadir / 'UNITTEST_BTC-1m.json' - _backup_file(file, copy_file=True) load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC']) assert file.is_file() assert not log_has( 'Download history data for pair: "UNITTEST/BTC", interval: 1m ' 'and store in None.', caplog ) - _clean_test_file(file) def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: @@ -112,17 +110,17 @@ def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, - default_conf, testdatadir) -> None: + default_conf, tmpdir) -> None: """ Test load_pair_history() with 1 min timeframe """ + tmpdir1 = Path(tmpdir) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) - file = testdatadir / 'MEME_BTC-1m.json' + file = tmpdir1 / 'MEME_BTC-1m.json' - _backup_file(file) # do not download a new pair if refresh_pairs isn't set - load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') + load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') assert not file.is_file() assert log_has( 'No history data for pair: "MEME/BTC", timeframe: 1m. ' @@ -130,15 +128,14 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, ) # download a new pair if refresh_pairs is set - refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'], + refresh_data(datadir=tmpdir1, timeframe='1m', pairs=['MEME/BTC'], exchange=exchange) - load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') + load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') assert file.is_file() assert log_has_re( 'Download history data for pair: "MEME/BTC", timeframe: 1m ' 'and store in .*', caplog ) - _clean_test_file(file) def test_testdata_path(testdatadir) -> None: @@ -231,26 +228,22 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: assert start_ts is None -def test_download_pair_history(ohlcv_history_list, mocker, default_conf, testdatadir) -> None: +def test_download_pair_history(ohlcv_history_list, mocker, default_conf, tmpdir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) - file1_1 = testdatadir / 'MEME_BTC-1m.json' - file1_5 = testdatadir / 'MEME_BTC-5m.json' - file2_1 = testdatadir / 'CFI_BTC-1m.json' - file2_5 = testdatadir / 'CFI_BTC-5m.json' - - _backup_file(file1_1) - _backup_file(file1_5) - _backup_file(file2_1) - _backup_file(file2_5) + tmpdir1 = Path(tmpdir) + file1_1 = tmpdir1 / 'MEME_BTC-1m.json' + file1_5 = tmpdir1 / 'MEME_BTC-5m.json' + file2_1 = tmpdir1 / 'CFI_BTC-1m.json' + file2_5 = tmpdir1 / 'CFI_BTC-5m.json' assert not file1_1.is_file() assert not file2_1.is_file() - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', timeframe='1m') - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='CFI/BTC', timeframe='1m') assert not exchange._pairs_last_refresh_time @@ -264,20 +257,16 @@ def test_download_pair_history(ohlcv_history_list, mocker, default_conf, testdat assert not file1_5.is_file() assert not file2_5.is_file() - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', timeframe='5m') - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='CFI/BTC', timeframe='5m') assert not exchange._pairs_last_refresh_time assert file1_5.is_file() assert file2_5.is_file() - # clean files freshly downloaded - _clean_test_file(file1_5) - _clean_test_file(file2_5) - def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: tick = [ @@ -294,24 +283,15 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: assert json_dump_mock.call_count == 2 -def test_download_backtesting_data_exception(ohlcv_history, mocker, caplog, - default_conf, testdatadir) -> None: +def test_download_backtesting_data_exception(mocker, caplog, default_conf, tmpdir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', side_effect=Exception('File Error')) - + tmpdir1 = Path(tmpdir) exchange = get_patched_exchange(mocker, default_conf) - file1_1 = testdatadir / 'MEME_BTC-1m.json' - file1_5 = testdatadir / 'MEME_BTC-5m.json' - _backup_file(file1_1) - _backup_file(file1_5) - - assert not _download_pair_history(datadir=testdatadir, exchange=exchange, + assert not _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', timeframe='1m') - # clean files freshly downloaded - _clean_test_file(file1_1) - _clean_test_file(file1_5) assert log_has('Failed to download history data for pair: "MEME/BTC", timeframe: 1m.', caplog) @@ -528,15 +508,15 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes assert log_has("Skipping pair XRP/ETH...", caplog) -def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog) -> None: - +def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog, + tmpdir) -> None: + tmpdir1 = Path(tmpdir) ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history)) mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', ght_mock) exchange = get_patched_exchange(mocker, default_conf) - file1 = testdatadir / 'ETH_BTC-trades.json.gz' - data_handler = get_datahandler(testdatadir, data_format='jsongz') - _backup_file(file1) + file1 = tmpdir1 / 'ETH_BTC-trades.json.gz' + data_handler = get_datahandler(tmpdir1, data_format='jsongz') assert not file1.is_file() @@ -557,8 +537,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time2 - 5 assert ght_mock.call_args_list[0][1]['from_id'] is not None - # clean files freshly downloaded - _clean_test_file(file1) + file1.unlink() mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', MagicMock(side_effect=ValueError)) @@ -567,9 +546,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) - file2 = testdatadir / 'XRP_ETH-trades.json.gz' - - _backup_file(file2, True) + file2 = tmpdir1 / 'XRP_ETH-trades.json.gz' + copyfile(testdatadir / file2.name, file2) ght_mock.reset_mock() mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', @@ -589,38 +567,37 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad _clean_test_file(file2) -def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): - +def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog): + tmpdir1 = Path(tmpdir) pair = 'XRP/ETH' - file1 = testdatadir / 'XRP_ETH-1m.json' - file5 = testdatadir / 'XRP_ETH-5m.json' - # Compare downloaded dataset with converted dataset - dfbak_1m = load_pair_history(datadir=testdatadir, timeframe="1m", pair=pair) - dfbak_5m = load_pair_history(datadir=testdatadir, timeframe="5m", pair=pair) + file1 = tmpdir1 / 'XRP_ETH-1m.json' + file5 = tmpdir1 / 'XRP_ETH-5m.json' + filetrades = tmpdir1 / 'XRP_ETH-trades.json.gz' + copyfile(testdatadir / file1.name, file1) + copyfile(testdatadir / file5.name, file5) + copyfile(testdatadir / filetrades.name, filetrades) - _backup_file(file1, copy_file=True) - _backup_file(file5) + # Compare downloaded dataset with converted dataset + dfbak_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) + dfbak_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) tr = TimeRange.parse_timerange('20191011-20191012') convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'], - datadir=testdatadir, timerange=tr, erase=True) + datadir=tmpdir1, timerange=tr, erase=True) assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog) # Load new data - df_1m = load_pair_history(datadir=testdatadir, timeframe="1m", pair=pair) - df_5m = load_pair_history(datadir=testdatadir, timeframe="5m", pair=pair) + df_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) + df_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) assert df_1m.equals(dfbak_1m) assert df_5m.equals(dfbak_5m) - _clean_test_file(file1) - _clean_test_file(file5) - assert not log_has('Could not convert NoDatapair to OHLCV.', caplog) convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], - datadir=testdatadir, timerange=tr, erase=True) + datadir=tmpdir1, timerange=tr, erase=True) assert log_has('Could not convert NoDatapair to OHLCV.', caplog) @@ -752,15 +729,17 @@ def test_hdf5datahandler_trades_load(testdatadir): assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0 -def test_hdf5datahandler_trades_store(testdatadir): +def test_hdf5datahandler_trades_store(testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) dh = HDF5DataHandler(testdatadir) trades = dh.trades_load('XRP/ETH') - dh.trades_store('XRP/NEW', trades) - file = testdatadir / 'XRP_NEW-trades.h5' + dh1 = HDF5DataHandler(tmpdir1) + dh1.trades_store('XRP/NEW', trades) + file = tmpdir1 / 'XRP_NEW-trades.h5' assert file.is_file() # Load trades back - trades_new = dh.trades_load('XRP/NEW') + trades_new = dh1.trades_load('XRP/NEW') assert len(trades_new) == len(trades) assert trades[0][0] == trades_new[0][0] @@ -778,8 +757,6 @@ def test_hdf5datahandler_trades_store(testdatadir): assert trades[-1][5] == trades_new[-1][5] assert trades[-1][6] == trades_new[-1][6] - _clean_test_file(file) - def test_hdf5datahandler_trades_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) @@ -793,16 +770,18 @@ def test_hdf5datahandler_trades_purge(mocker, testdatadir): assert unlinkmock.call_count == 1 -def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir): +def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) dh = HDF5DataHandler(testdatadir) ohlcv = dh.ohlcv_load('UNITTEST/BTC', '5m') assert isinstance(ohlcv, DataFrame) assert len(ohlcv) > 0 - file = testdatadir / 'UNITTEST_NEW-5m.h5' + file = tmpdir1 / 'UNITTEST_NEW-5m.h5' assert not file.is_file() - dh.ohlcv_store('UNITTEST/NEW', '5m', ohlcv) + dh1 = HDF5DataHandler(tmpdir1) + dh1.ohlcv_store('UNITTEST/NEW', '5m', ohlcv) assert file.is_file() assert not ohlcv[ohlcv['date'] < '2018-01-15'].empty @@ -812,14 +791,12 @@ def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir): # Call private function to ensure timerange is filtered in hdf5 ohlcv = dh._ohlcv_load('UNITTEST/BTC', '5m', timerange) - ohlcv1 = dh._ohlcv_load('UNITTEST/NEW', '5m', timerange) + ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', '5m', timerange) assert len(ohlcv) == len(ohlcv1) assert ohlcv.equals(ohlcv1) assert ohlcv[ohlcv['date'] < '2018-01-15'].empty assert ohlcv[ohlcv['date'] > '2018-01-19'].empty - _clean_test_file(file) - # Try loading inexisting file ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', '5m') assert ohlcv.empty diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index c30bce6a4..0655b3a0f 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -12,8 +12,8 @@ from pandas import DataFrame, to_datetime from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo +from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException -from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset) @@ -266,7 +266,7 @@ def test_edge_heartbeat_calculate(mocker, edge_conf): # should not recalculate if heartbeat not reached edge._last_updated = arrow.utcnow().int_timestamp - heartbeat + 1 - assert edge.calculate() is False + assert edge.calculate(edge_conf['exchange']['pair_whitelist']) is False def mocked_load_data(datadir, pairs=[], timeframe='0m', @@ -310,7 +310,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf): mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) - assert edge.calculate() + assert edge.calculate(edge_conf['exchange']['pair_whitelist']) assert len(edge._cached_pairs) == 2 assert edge._last_updated <= arrow.utcnow().int_timestamp + 2 @@ -322,7 +322,7 @@ def test_edge_process_no_data(mocker, edge_conf, caplog): mocker.patch('freqtrade.edge.edge_positioning.load_data', MagicMock(return_value={})) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) - assert not edge.calculate() + assert not edge.calculate(edge_conf['exchange']['pair_whitelist']) assert len(edge._cached_pairs) == 0 assert log_has("No data found. Edge is stopped ...", caplog) assert edge._last_updated == 0 @@ -330,18 +330,37 @@ def test_edge_process_no_data(mocker, edge_conf, caplog): def test_edge_process_no_trades(mocker, edge_conf, caplog): freqtrade = get_patched_freqtradebot(mocker, edge_conf) - mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) - mocker.patch('freqtrade.edge.edge_positioning.refresh_data', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.get_fee', return_value=0.001) + mocker.patch('freqtrade.edge.edge_positioning.refresh_data', ) mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data) # Return empty - mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', MagicMock(return_value=[])) + mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', return_value=[]) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) - assert not edge.calculate() + assert not edge.calculate(edge_conf['exchange']['pair_whitelist']) assert len(edge._cached_pairs) == 0 assert log_has("No trades found.", caplog) +def test_edge_process_no_pairs(mocker, edge_conf, caplog): + edge_conf['exchange']['pair_whitelist'] = [] + mocker.patch('freqtrade.freqtradebot.validate_config_consistency') + + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + fee_mock = mocker.patch('freqtrade.exchange.Exchange.get_fee', return_value=0.001) + mocker.patch('freqtrade.edge.edge_positioning.refresh_data') + mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data) + # Return empty + mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', return_value=[]) + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + assert fee_mock.call_count == 0 + assert edge.fee is None + + assert not edge.calculate(['XRP/USDT']) + assert fee_mock.call_count == 1 + assert edge.fee == 0.001 + + def test_edge_init_error(mocker, edge_conf,): edge_conf['stake_amount'] = 0.5 mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 8e1d074aa..dce10da84 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -5,10 +5,12 @@ However, these tests should give a good idea to determine if a new exchange is suitable to run with freqtrade. """ +from datetime import datetime, timedelta, timezone from pathlib import Path import pytest +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_default_conf @@ -34,7 +36,12 @@ EXCHANGES = { 'pair': 'BTC/USDT', 'hasQuoteVolume': True, 'timeframe': '5m', - } + }, + 'kucoin': { + 'pair': 'BTC/USDT', + 'hasQuoteVolume': True, + 'timeframe': '5m', + }, } @@ -42,6 +49,7 @@ EXCHANGES = { def exchange_conf(): config = get_default_conf((Path(__file__).parent / "testdata").resolve()) config['exchange']['pair_whitelist'] = [] + config['dry_run'] = False return config @@ -97,14 +105,16 @@ class TestCCXTExchange(): assert 'asks' in l2 assert 'bids' in l2 l2_limit_range = exchange._ft_has['l2_limit_range'] + l2_limit_range_required = exchange._ft_has['l2_limit_range_required'] for val in [1, 2, 5, 25, 100]: l2 = exchange.fetch_l2_order_book(pair, val) if not l2_limit_range or val in l2_limit_range: assert len(l2['asks']) == val assert len(l2['bids']) == val else: - next_limit = exchange.get_next_limit_in_list(val, l2_limit_range) - if next_limit > 200: + next_limit = exchange.get_next_limit_in_list( + val, l2_limit_range, l2_limit_range_required) + if next_limit is None or next_limit > 200: # Large orderbook sizes can be a problem for some exchanges (bitrex ...) assert len(l2['asks']) > 200 assert len(l2['asks']) > 200 @@ -122,7 +132,10 @@ class TestCCXTExchange(): assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf)) # assert len(exchange.klines(pair_tf)) > 200 # Assume 90% uptime ... - assert len(exchange.klines(pair_tf)) > exchange._ohlcv_candle_limit * 0.90 + assert len(exchange.klines(pair_tf)) > exchange.ohlcv_candle_limit(timeframe) * 0.90 + # Check if last-timeframe is within the last 2 intervals + now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2)) + assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) # TODO: tests fetch_trades (?) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f35a84725..5fa94e6c1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1,6 +1,7 @@ import copy import logging from datetime import datetime, timedelta, timezone +from math import isclose from random import randint from unittest.mock import MagicMock, Mock, PropertyMock, patch @@ -10,7 +11,7 @@ import pytest from pandas import DataFrame from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException, - OperationalException, TemporaryError) + OperationalException, PricingError, TemporaryError) from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff) @@ -18,21 +19,13 @@ from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_next_date, timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.resolvers.exchange_resolver import ExchangeResolver -from tests.conftest import get_patched_exchange, log_has, log_has_re +from tests.conftest import get_mock_coro, get_patched_exchange, log_has, log_has_re # Make sure to always keep one exchange here which is NOT subclassed!! EXCHANGES = ['bittrex', 'binance', 'kraken', 'ftx'] -# Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines -def get_mock_coro(return_value): - async def mock_coro(*args, **kwargs): - return return_value - - return Mock(wraps=mock_coro) - - def ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, fun, mock_ccxt_fun, retries=API_RETRY_COUNT + 1, **kwargs): @@ -378,7 +371,7 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: PropertyMock(return_value=markets) ) result = exchange.get_min_pair_stake_amount('ETH/BTC', 1, stoploss) - assert result == 2 / 0.9 + assert isclose(result, 2 * (1+0.05) / (1-abs(stoploss))) # min amount is set markets["ETH/BTC"]["limits"] = { @@ -390,7 +383,7 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: PropertyMock(return_value=markets) ) result = exchange.get_min_pair_stake_amount('ETH/BTC', 2, stoploss) - assert result == 2 * 2 / 0.9 + assert isclose(result, 2 * 2 * (1+0.05) / (1-abs(stoploss))) # min amount and cost are set (cost is minimal) markets["ETH/BTC"]["limits"] = { @@ -402,7 +395,7 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: PropertyMock(return_value=markets) ) result = exchange.get_min_pair_stake_amount('ETH/BTC', 2, stoploss) - assert result == max(2, 2 * 2) / 0.9 + assert isclose(result, max(2, 2 * 2) * (1+0.05) / (1-abs(stoploss))) # min amount and cost are set (amount is minial) markets["ETH/BTC"]["limits"] = { @@ -414,7 +407,14 @@ def test_get_min_pair_stake_amount(mocker, default_conf) -> None: PropertyMock(return_value=markets) ) result = exchange.get_min_pair_stake_amount('ETH/BTC', 2, stoploss) - assert result == max(8, 2 * 2) / 0.9 + assert isclose(result, max(8, 2 * 2) * (1+0.05) / (1-abs(stoploss))) + + result = exchange.get_min_pair_stake_amount('ETH/BTC', 2, -0.4) + assert isclose(result, max(8, 2 * 2) * 1.5) + + # Really big stoploss + result = exchange.get_min_pair_stake_amount('ETH/BTC', 2, -1) + assert isclose(result, max(8, 2 * 2) * 1.5) def test_get_min_pair_stake_amount_real_data(mocker, default_conf) -> None: @@ -432,7 +432,10 @@ def test_get_min_pair_stake_amount_real_data(mocker, default_conf) -> None: PropertyMock(return_value=markets) ) result = exchange.get_min_pair_stake_amount('ETH/BTC', 0.020405, stoploss) - assert round(result, 8) == round(max(0.0001, 0.001 * 0.020405) / 0.9, 8) + assert round(result, 8) == round( + max(0.0001, 0.001 * 0.020405) * (1+0.05) / (1-abs(stoploss)), + 8 + ) def test_set_sandbox(default_conf, mocker): @@ -498,7 +501,7 @@ def test__load_markets(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._load_async_markets') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) - assert log_has('Unable to initialize markets. Reason: SomeError', caplog) + assert log_has('Unable to initialize markets.', caplog) expected_return = {'ETH/BTC': 'available'} api_mock = MagicMock() @@ -931,11 +934,11 @@ def test_exchange_has(default_conf, mocker): ("sell") ]) @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_dry_run_order(default_conf, mocker, side, exchange_name): +def test_create_dry_run_order(default_conf, mocker, side, exchange_name): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) - order = exchange.dry_run_order( + order = exchange.create_dry_run_order( pair='ETH/BTC', ordertype='limit', side=side, amount=1, rate=200) assert 'id' in order assert f'dry_run_{side}_' in order["id"] @@ -1245,44 +1248,6 @@ def test_sell_considers_time_in_force(default_conf, mocker, exchange_name): assert "timeInForce" not in api_mock.create_order.call_args[0][5] -def test_get_balance_dry_run(default_conf, mocker): - default_conf['dry_run'] = True - default_conf['dry_run_wallet'] = 999.9 - - exchange = get_patched_exchange(mocker, default_conf) - assert exchange.get_balance(currency='BTC') == 999.9 - - -@pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_balance_prod(default_conf, mocker, exchange_name): - api_mock = MagicMock() - api_mock.fetch_balance = MagicMock(return_value={'BTC': {'free': 123.4, 'total': 123.4}}) - default_conf['dry_run'] = False - - exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - - assert exchange.get_balance(currency='BTC') == 123.4 - - with pytest.raises(OperationalException): - api_mock.fetch_balance = MagicMock(side_effect=ccxt.BaseError("Unknown error")) - exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - - exchange.get_balance(currency='BTC') - - with pytest.raises(TemporaryError, match=r'.*balance due to malformed exchange response:.*'): - exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - mocker.patch('freqtrade.exchange.Exchange.get_balances', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Kraken.get_balances', MagicMock(return_value={})) - exchange.get_balance(currency='BTC') - - -@pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_balances_dry_run(default_conf, mocker, exchange_name): - default_conf['dry_run'] = True - exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) - assert exchange.get_balances() == {} - - @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_get_balances_prod(default_conf, mocker, exchange_name): balance_item = { @@ -1334,6 +1299,16 @@ def test_get_tickers(default_conf, mocker, exchange_name): assert tickers['ETH/BTC']['ask'] == 1 assert tickers['BCH/BTC']['bid'] == 0.6 assert tickers['BCH/BTC']['ask'] == 0.5 + assert api_mock.fetch_tickers.call_count == 1 + + api_mock.fetch_tickers.reset_mock() + + # Cached ticker should not call api again + tickers2 = exchange.get_tickers(cached=True) + assert tickers2 == tickers + assert api_mock.fetch_tickers.call_count == 0 + tickers2 = exchange.get_tickers(cached=False) + assert api_mock.fetch_tickers.call_count == 1 ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, "get_tickers", "fetch_tickers") @@ -1417,7 +1392,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls - since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8 + since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 ret = exchange.get_historic_ohlcv(pair, "5m", int(( arrow.utcnow().int_timestamp - since) * 1000)) @@ -1473,7 +1448,7 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls - since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8 + since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 ret = exchange.get_historic_ohlcv_as_df(pair, "5m", int(( arrow.utcnow().int_timestamp - since) * 1000)) @@ -1656,6 +1631,9 @@ def test_get_next_limit_in_list(): # Going over the limit ... assert Exchange.get_next_limit_in_list(1001, limit_range) == 1000 assert Exchange.get_next_limit_in_list(2000, limit_range) == 1000 + # Without required range + assert Exchange.get_next_limit_in_list(2000, limit_range, False) is None + assert Exchange.get_next_limit_in_list(15, limit_range, False) == 20 assert Exchange.get_next_limit_in_list(21, None) == 21 assert Exchange.get_next_limit_in_list(100, None) == 100 @@ -1706,6 +1684,152 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50) +@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ + ('ask', 20, 19, 10, 0.0, 20), # Full ask side + ('ask', 20, 19, 10, 1.0, 10), # Full last side + ('ask', 20, 19, 10, 0.5, 15), # Between ask and last + ('ask', 20, 19, 10, 0.7, 13), # Between ask and last + ('ask', 20, 19, 10, 0.3, 17), # Between ask and last + ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask + ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask + ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask + ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask + ('ask', 4, 5, None, 1, 4), # last not available - uses ask + ('ask', 4, 5, None, 0, 4), # last not available - uses ask + ('bid', 21, 20, 10, 0.0, 20), # Full bid side + ('bid', 21, 20, 10, 1.0, 10), # Full last side + ('bid', 21, 20, 10, 0.5, 15), # Between bid and last + ('bid', 21, 20, 10, 0.7, 13), # Between bid and last + ('bid', 21, 20, 10, 0.3, 17), # Between bid and last + ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid + ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid + ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid + ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid + ('bid', 6, 5, None, 1, 5), # last not available - uses bid + ('bid', 6, 5, None, 0, 5), # last not available - uses bid +]) +def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, + last, last_ab, expected) -> None: + caplog.set_level(logging.DEBUG) + default_conf['bid_strategy']['ask_last_balance'] = last_ab + default_conf['bid_strategy']['price_side'] = side + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': ask, 'last': last, 'bid': bid}) + + assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + assert exchange.get_buy_rate('ETH/BTC', False) == expected + assert log_has("Using cached buy rate for ETH/BTC.", caplog) + # Running a 2nd time with Refresh on! + caplog.clear() + assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + +@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', [ + ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side + ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side + ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat + ('bid', 12.0, 11.2, 10.5, 0.0, 11.2), # Last smaller than bid + ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid + ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid + ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), + ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side + ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side + ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat + ('ask', 12.2, 11.2, 10.5, 0.0, 12.2), # Last smaller than ask + ('ask', 12.0, 11.0, 10.5, 1.0, 12.0), # Last smaller than ask - uses ask + ('ask', 12.0, 11.2, 10.5, 0.5, 12.0), # Last smaller than ask - uses ask + ('ask', 10.0, 11.0, 11.0, 0.0, 10.0), + ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), + ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), + ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), +]) +def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, + last, last_ab, expected) -> None: + caplog.set_level(logging.DEBUG) + + default_conf['ask_strategy']['price_side'] = side + default_conf['ask_strategy']['bid_last_balance'] = last_ab + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': ask, 'bid': bid, 'last': last}) + pair = "ETH/BTC" + + # Test regular mode + exchange = get_patched_exchange(mocker, default_conf) + rate = exchange.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) + assert isinstance(rate, float) + assert rate == expected + # Use caching + rate = exchange.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + +@pytest.mark.parametrize('side,expected', [ + ('bid', 0.043936), # Value from order_book_l2 fiture - bids side + ('ask', 0.043949), # Value from order_book_l2 fiture - asks side +]) +def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2): + caplog.set_level(logging.DEBUG) + # Test orderbook mode + default_conf['ask_strategy']['price_side'] = side + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) + exchange = get_patched_exchange(mocker, default_conf) + rate = exchange.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) + assert isinstance(rate, float) + assert rate == expected + rate = exchange.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + +def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog): + # Test orderbook mode + default_conf['ask_strategy']['price_side'] = 'ask' + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + pair = "ETH/BTC" + # Test What happens if the exchange returns an empty orderbook. + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', + return_value={'bids': [[]], 'asks': [[]]}) + exchange = get_patched_exchange(mocker, default_conf) + with pytest.raises(PricingError): + exchange.get_sell_rate(pair, True) + assert log_has("Sell Price at location from orderbook could not be determined.", caplog) + + +def test_get_sell_rate_exception(default_conf, mocker, caplog): + # Ticker on one side can be empty in certain circumstances. + default_conf['ask_strategy']['price_side'] = 'ask' + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': None, 'bid': 0.12, 'last': None}) + exchange = get_patched_exchange(mocker, default_conf) + with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): + exchange.get_sell_rate(pair, True) + + exchange._config['ask_strategy']['price_side'] = 'bid' + assert exchange.get_sell_rate(pair, True) == 0.12 + # Reverse sides + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': 0.13, 'bid': None, 'last': None}) + with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): + exchange.get_sell_rate(pair, True) + + exchange._config['ask_strategy']['price_side'] = 'ask' + assert exchange.get_sell_rate(pair, True) == 0.13 + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: @@ -2106,6 +2230,46 @@ def test_cancel_stoploss_order(default_conf, mocker, exchange_name): order_id='_', pair='TKN/BTC') +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_cancel_stoploss_order_with_result(default_conf, mocker, exchange_name): + default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', return_value={'for': 123}) + mocker.patch('freqtrade.exchange.Ftx.fetch_stoploss_order', return_value={'for': 123}) + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + + mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', + return_value={'fee': {}, 'status': 'canceled', 'amount': 1234}) + mocker.patch('freqtrade.exchange.Ftx.cancel_stoploss_order', + return_value={'fee': {}, 'status': 'canceled', 'amount': 1234}) + co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) + assert co == {'fee': {}, 'status': 'canceled', 'amount': 1234} + + mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', + return_value='canceled') + mocker.patch('freqtrade.exchange.Ftx.cancel_stoploss_order', + return_value='canceled') + # Fall back to fetch_stoploss_order + co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) + assert co == {'for': 123} + + mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', + side_effect=InvalidOrderException("")) + mocker.patch('freqtrade.exchange.Ftx.fetch_stoploss_order', + side_effect=InvalidOrderException("")) + + co = exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=555) + assert co['amount'] == 555 + assert co == {'fee': {}, 'status': 'canceled', 'amount': 555, 'info': {}} + + with pytest.raises(InvalidOrderException): + mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', + side_effect=InvalidOrderException("Did not find order")) + mocker.patch('freqtrade.exchange.Ftx.cancel_stoploss_order', + side_effect=InvalidOrderException("Did not find order")) + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + exchange.cancel_stoploss_order_with_result(order_id='_', pair='TKN/BTC', amount=123) + + @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_fetch_order(default_conf, mocker, exchange_name): default_conf['dry_run'] = True @@ -2276,12 +2440,20 @@ def test_get_fee(default_conf, mocker, exchange_name): 'cost': 0.05 }) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + exchange._config.pop('fee', None) assert exchange.get_fee('ETH/BTC') == 0.025 + assert api_mock.calculate_fee.call_count == 1 ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, 'get_fee', 'calculate_fee', symbol="ETH/BTC") + api_mock.calculate_fee.reset_mock() + exchange._config['fee'] = 0.001 + + assert exchange.get_fee('ETH/BTC') == 0.001 + assert api_mock.calculate_fee.call_count == 0 + def test_stoploss_order_unsupported_exchange(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, id='bittrex') @@ -2418,6 +2590,19 @@ def test_get_markets_error(default_conf, mocker): ex.get_markets('LTC', 'USDT', True, False) +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_ohlcv_candle_limit(default_conf, mocker, exchange_name): + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + timeframes = ('1m', '5m', '1h') + expected = exchange._ft_has['ohlcv_candle_limit'] + for timeframe in timeframes: + if 'ohlcv_candle_limit_per_timeframe' in exchange._ft_has: + expected = exchange._ft_has['ohlcv_candle_limit_per_timeframe'][timeframe] + # This should only run for bittrex + assert exchange_name == 'bittrex' + assert exchange.ohlcv_candle_limit(timeframe) == expected + + def test_timeframe_to_minutes(): assert timeframe_to_minutes("5m") == 5 assert timeframe_to_minutes("10m") == 10 diff --git a/tests/exchange/test_ftx.py b/tests/exchange/test_ftx.py index 17cfb26fa..3794bb79c 100644 --- a/tests/exchange/test_ftx.py +++ b/tests/exchange/test_ftx.py @@ -39,8 +39,9 @@ def test_stoploss_order_ftx(default_conf, mocker): assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_ORDERTYPE assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 - assert api_mock.create_order.call_args_list[0][1]['price'] == 190 assert 'orderPrice' not in api_mock.create_order.call_args_list[0][1]['params'] + assert 'stopPrice' in api_mock.create_order.call_args_list[0][1]['params'] + assert api_mock.create_order.call_args_list[0][1]['params']['stopPrice'] == 190 assert api_mock.create_order.call_count == 1 @@ -55,8 +56,8 @@ def test_stoploss_order_ftx(default_conf, mocker): assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_ORDERTYPE assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 - assert api_mock.create_order.call_args_list[0][1]['price'] == 220 assert 'orderPrice' not in api_mock.create_order.call_args_list[0][1]['params'] + assert api_mock.create_order.call_args_list[0][1]['params']['stopPrice'] == 220 api_mock.create_order.reset_mock() order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, @@ -69,9 +70,9 @@ def test_stoploss_order_ftx(default_conf, mocker): assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_ORDERTYPE assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 - assert api_mock.create_order.call_args_list[0][1]['price'] == 220 assert 'orderPrice' in api_mock.create_order.call_args_list[0][1]['params'] assert api_mock.create_order.call_args_list[0][1]['params']['orderPrice'] == 217.8 + assert api_mock.create_order.call_args_list[0][1]['params']['stopPrice'] == 220 # test exception handling with pytest.raises(DependencyException): @@ -124,7 +125,7 @@ def test_stoploss_adjust_ftx(mocker, default_conf): assert not exchange.stoploss_adjust(1501, order) -def test_fetch_stoploss_order(default_conf, mocker): +def test_fetch_stoploss_order(default_conf, mocker, limit_sell_order): default_conf['dry_run'] = True order = MagicMock() order.myid = 123 @@ -146,6 +147,17 @@ def test_fetch_stoploss_order(default_conf, mocker): with pytest.raises(InvalidOrderException, match=r"Could not get stoploss order for id X"): exchange.fetch_stoploss_order('X', 'TKN/BTC')['status'] + api_mock.fetch_orders = MagicMock(return_value=[{'id': 'X', 'status': 'closed'}]) + api_mock.fetch_order = MagicMock(return_value=limit_sell_order) + + resp = exchange.fetch_stoploss_order('X', 'TKN/BTC') + assert resp + assert api_mock.fetch_order.call_count == 1 + assert resp['id_stop'] == 'mocked_limit_sell' + assert resp['id'] == 'X' + assert resp['type'] == 'stop' + assert resp['status_stop'] == 'triggered' + with pytest.raises(InvalidOrderException): api_mock.fetch_orders = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id='ftx') @@ -156,3 +168,26 @@ def test_fetch_stoploss_order(default_conf, mocker): 'fetch_stoploss_order', 'fetch_orders', retries=API_FETCH_ORDER_RETRY_COUNT + 1, order_id='_', pair='TKN/BTC') + + +def test_get_order_id(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf, id='ftx') + order = { + 'type': STOPLOSS_ORDERTYPE, + 'price': 1500, + 'id': '1111', + 'id_stop': '1234', + 'info': { + } + } + assert exchange.get_order_id_conditional(order) == '1234' + + order = { + 'type': 'limit', + 'price': 1500, + 'id': '1111', + 'id_stop': '1234', + 'info': { + } + } + assert exchange.get_order_id_conditional(order) == '1111' diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index 97f428e2f..ed22cde92 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -90,6 +90,7 @@ def test_get_balances_prod(default_conf, mocker): '3ST': balance_item.copy(), '4ST': balance_item.copy(), 'EUR': balance_item.copy(), + 'timestamp': 123123 }) kraken_open_orders = [{'symbol': '1ST/EUR', 'type': 'limit', @@ -138,7 +139,7 @@ def test_get_balances_prod(default_conf, mocker): default_conf['dry_run'] = False exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") balances = exchange.get_balances() - assert len(balances) == 5 + assert len(balances) == 6 assert balances['1ST']['free'] == 9.0 assert balances['1ST']['total'] == 10.0 diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index 306850ff6..7cca2b1a8 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -3,8 +3,8 @@ from typing import Dict, List, NamedTuple, Optional import arrow from pandas import DataFrame +from freqtrade.enums import SellType from freqtrade.exchange import timeframe_to_minutes -from freqtrade.strategy.interface import SellType tests_start_time = arrow.get(2018, 10, 3) diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index df6f22e01..a7fd238d1 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -5,8 +5,8 @@ from pathlib import Path import pandas as pd import pytest +from freqtrade.enums import RunMode, SellType from freqtrade.optimize.hyperopt import Hyperopt -from freqtrade.strategy.interface import SellType from tests.conftest import patch_exchange @@ -14,6 +14,8 @@ from tests.conftest import patch_exchange def hyperopt_conf(default_conf): hyperconf = deepcopy(default_conf) hyperconf.update({ + 'datadir': Path(default_conf['datadir']), + 'runmode': RunMode.HYPEROPT, 'hyperopt': 'DefaultHyperOpt', 'hyperopt_loss': 'ShortTradeDurHyperOptLoss', 'hyperopt_path': str(Path(__file__).parent / 'hyperopts'), @@ -21,6 +23,7 @@ def hyperopt_conf(default_conf): 'timerange': None, 'spaces': ['default'], 'hyperopt_jobs': 1, + 'hyperopt_min_trades': 1, }) return hyperconf diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index daf7c2053..e5b969383 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -1,12 +1,11 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, C0330, unused-argument import logging -from unittest.mock import MagicMock import pytest from freqtrade.data.history import get_timerange +from freqtrade.enums import SellType from freqtrade.optimize.backtesting import Backtesting -from freqtrade.strategy.interface import SellType from tests.conftest import patch_exchange from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset, tests_timeframe) @@ -186,7 +185,7 @@ tc11 = BTContainer(data=[ [0, 5000, 5050, 4950, 5000, 6172, 1, 0], [1, 5000, 5050, 4950, 5100, 6172, 0, 0], [2, 5100, 5251, 5100, 5100, 6172, 0, 0], - [3, 4850, 5050, 4650, 4750, 6172, 0, 0], + [3, 5000, 5150, 4650, 4750, 6172, 0, 0], [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], stop_loss=-0.10, roi={"0": 0.10}, profit_perc=0.019, trailing_stop=True, trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, @@ -269,7 +268,7 @@ tc16 = BTContainer(data=[ # Test 17: Buy, hold for 120 mins, then forcesell using roi=-1 # Causes negative profit even though sell-reason is ROI. # stop-loss: 10%, ROI: 10% (should not apply), -100% after 100 minutes (limits trade duration) -# Uses open as sell-rate (special case) - since the roi-time is a multiple of the ticker interval. +# Uses open as sell-rate (special case) - since the roi-time is a multiple of the timeframe. tc17 = BTContainer(data=[ # D O H L C V B S [0, 5000, 5025, 4975, 4987, 6172, 1, 0], @@ -441,6 +440,23 @@ tc27 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=4)] ) +# Test 28: trailing_stop should raise so candle 3 causes a stoploss +# Same case than tc11 - but candle 3 "gaps down" - the stoploss will be above the candle, +# therefore "open" will be used +# stop-loss: 10%, ROI: 10% (should not apply), stoploss adjusted candle 2 +tc28 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 5100, 5100, 6172, 0, 0], + [3, 4850, 5050, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.03, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)] +) + TESTS = [ tc0, tc1, @@ -470,6 +486,7 @@ TESTS = [ tc25, tc26, tc27, + tc28, ] @@ -489,10 +506,12 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: default_conf["trailing_stop_positive_offset"] = data.trailing_stop_positive_offset default_conf["ask_strategy"] = {"use_sell_signal": data.use_sell_signal} - mocker.patch("freqtrade.exchange.Exchange.get_fee", MagicMock(return_value=0.0)) + mocker.patch("freqtrade.exchange.Exchange.get_fee", return_value=0.0) + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) patch_exchange(mocker) frame = _build_backtest_dataframe(data.data) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = lambda a, m: frame backtesting.strategy.advise_sell = lambda a, m: frame caplog.set_level(logging.DEBUG) @@ -501,19 +520,19 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: # Dummy data as we mock the analyze functions data_processed = {pair: frame.copy()} min_date, max_date = get_timerange({pair: frame}) - results = backtesting.backtest( + result = backtesting.backtest( processed=data_processed, - stake_amount=default_conf['stake_amount'], start_date=min_date, end_date=max_date, max_open_trades=10, ) + results = result['results'] assert len(results) == len(data.trades) assert round(results["profit_ratio"].sum(), 3) == round(data.profit_perc, 3) for c, trade in enumerate(data.trades): res = results.iloc[c] - assert res.sell_reason == trade.sell_reason + assert res.sell_reason == trade.sell_reason.value assert res.open_date == _get_frame_time_from_offset(trade.open_tick) assert res.close_date == _get_frame_time_from_offset(trade.close_tick) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index c8d4338af..7387c8865 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -9,7 +9,6 @@ import pandas as pd import pytest from arrow import Arrow -from freqtrade import constants from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting from freqtrade.configuration import TimeRange from freqtrade.data import history @@ -17,11 +16,11 @@ from freqtrade.data.btanalysis import BT_DATA_COLUMNS, evaluate_result_multi from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange +from freqtrade.enums import RunMode, SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.optimize.backtesting import Backtesting +from freqtrade.persistence import LocalTrade from freqtrade.resolvers import StrategyResolver -from freqtrade.state import RunMode -from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -83,6 +82,7 @@ def simple_backtest(config, contour, mocker, testdatadir) -> None: patch_exchange(mocker) config['timeframe'] = '1m' backtesting = Backtesting(config) + backtesting._set_strategy(backtesting.strategylist[0]) data = load_data_test(contour, testdatadir) processed = backtesting.strategy.ohlcvdata_to_dataframe(data) @@ -90,7 +90,6 @@ def simple_backtest(config, contour, mocker, testdatadir) -> None: assert isinstance(processed, dict) results = backtesting.backtest( processed=processed, - stake_amount=config['stake_amount'], start_date=min_date, end_date=max_date, max_open_trades=1, @@ -107,11 +106,11 @@ def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'): data = trim_dictlist(data, -201) patch_exchange(mocker) backtesting = Backtesting(conf) + backtesting._set_strategy(backtesting.strategylist[0]) processed = backtesting.strategy.ohlcvdata_to_dataframe(data) min_date, max_date = get_timerange(processed) return { 'processed': processed, - 'stake_amount': conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, 'max_open_trades': 10, @@ -233,8 +232,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --fee detected, setting fee to: {} ...'.format(config['fee']), caplog) -def test_setup_optimize_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: - default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT +def test_setup_optimize_configuration_stake_amount(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) @@ -242,9 +240,21 @@ def test_setup_optimize_configuration_unlimited_stake_amount(mocker, default_con 'backtesting', '--config', 'config.json', '--strategy', 'DefaultStrategy', + '--stake-amount', '1', + '--starting-balance', '2' ] - with pytest.raises(DependencyException, match=r'.`stake_amount`.*'): + conf = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) + assert isinstance(conf, dict) + + args = [ + 'backtesting', + '--config', 'config.json', + '--strategy', 'DefaultStrategy', + '--stake-amount', '1', + '--starting-balance', '0.5' + ] + with pytest.raises(OperationalException, match=r"Starting balance .* smaller .*"): setup_optimize_configuration(get_args(args), RunMode.BACKTEST) @@ -276,6 +286,7 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None: patch_exchange(mocker) get_fee = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) assert backtesting.config == default_conf assert backtesting.timeframe == '5m' assert callable(backtesting.strategy.ohlcvdata_to_dataframe) @@ -306,11 +317,13 @@ def test_data_with_fee(default_conf, mocker, testdatadir) -> None: fee_mock = mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.5)) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) assert backtesting.fee == 0.1234 assert fee_mock.call_count == 0 default_conf['fee'] = 0.0 backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) assert backtesting.fee == 0.0 assert fee_mock.call_count == 0 @@ -321,6 +334,7 @@ def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: data = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, fill_up_missing=True) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) processed = backtesting.strategy.ohlcvdata_to_dataframe(data) assert len(processed['UNITTEST/BTC']) == 102 @@ -352,12 +366,13 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: default_conf['timerange'] = '-1510694220' backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.bot_loop_start = MagicMock() backtesting.start() # check the logs, that will contain the backtest result exists = [ 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:59:00 (0 days)..' + 'up to 2017-11-14 22:59:00 (0 days).' ] for line in exists: assert log_has(line, caplog) @@ -384,6 +399,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> default_conf['timerange'] = '20180101-20180102' backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) with pytest.raises(OperationalException, match='No data found. Terminating.'): backtesting.start() @@ -448,25 +464,73 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti Backtesting(default_conf) +def test_backtest__enter_trade(default_conf, fee, mocker) -> None: + default_conf['ask_strategy']['use_sell_signal'] = False + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) + patch_exchange(mocker) + default_conf['stake_amount'] = 'unlimited' + default_conf['max_open_trades'] = 2 + backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) + pair = 'UNITTEST/BTC' + row = [ + pd.Timestamp(year=2020, month=1, day=1, hour=5, minute=0), + 1, # Sell + 0.001, # Open + 0.0011, # Close + 0, # Sell + 0.00099, # Low + 0.0012, # High + ] + trade = backtesting._enter_trade(pair, row=row) + assert isinstance(trade, LocalTrade) + assert trade.stake_amount == 495 + + # Fake 2 trades, so there's not enough amount for the next trade left. + LocalTrade.trades_open.append(trade) + LocalTrade.trades_open.append(trade) + trade = backtesting._enter_trade(pair, row=row) + assert trade is None + LocalTrade.trades_open.pop() + trade = backtesting._enter_trade(pair, row=row) + assert trade is not None + + # Stake-amount too high! + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=600.0) + + trade = backtesting._enter_trade(pair, row=row) + assert trade is None + + # Stake-amount throwing error + mocker.patch("freqtrade.wallets.Wallets.get_trade_stake_amount", + side_effect=DependencyException) + + trade = backtesting._enter_trade(pair, row=row) + assert trade is None + + def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: default_conf['ask_strategy']['use_sell_signal'] = False mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) patch_exchange(mocker) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) pair = 'UNITTEST/BTC' timerange = TimeRange('date', None, 1517227800, 0) data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], timerange=timerange) processed = backtesting.strategy.ohlcvdata_to_dataframe(data) min_date, max_date = get_timerange(processed) - results = backtesting.backtest( + result = backtesting.backtest( processed=processed, - stake_amount=default_conf['stake_amount'], start_date=min_date, end_date=max_date, max_open_trades=10, position_stacking=False, ) + results = result['results'] assert not results.empty assert len(results) == 2 @@ -486,7 +550,7 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: 'trade_duration': [235, 40], 'profit_ratio': [0.0, 0.0], 'profit_abs': [0.0, 0.0], - 'sell_reason': [SellType.ROI, SellType.ROI], + 'sell_reason': [SellType.ROI.value, SellType.ROI.value], 'initial_stop_loss_abs': [0.0940005, 0.09272236], 'initial_stop_loss_ratio': [-0.1, -0.1], 'stop_loss_abs': [0.0940005, 0.09272236], @@ -512,8 +576,10 @@ def test_backtest_one(default_conf, fee, mocker, testdatadir) -> None: def test_backtest_1min_timeframe(default_conf, fee, mocker, testdatadir) -> None: default_conf['ask_strategy']['use_sell_signal'] = False mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) patch_exchange(mocker) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) # Run a backtesting for an exiting 1min timeframe timerange = TimeRange.parse_timerange('1510688220-1510700340') @@ -523,19 +589,19 @@ def test_backtest_1min_timeframe(default_conf, fee, mocker, testdatadir) -> None min_date, max_date = get_timerange(processed) results = backtesting.backtest( processed=processed, - stake_amount=default_conf['stake_amount'], start_date=min_date, end_date=max_date, max_open_trades=1, position_stacking=False, ) - assert not results.empty - assert len(results) == 1 + assert not results['results'].empty + assert len(results['results']) == 1 def test_processed(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) dict_of_tickerrows = load_data_test('raise', testdatadir) dataframes = backtesting.strategy.ohlcvdata_to_dataframe(dict_of_tickerrows) @@ -558,6 +624,7 @@ def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatad default_conf['enable_protections'] = True mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) tests = [ ['sine', 9], ['raise', 10], @@ -568,7 +635,7 @@ def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatad # While buy-signals are unrealistic, running backtesting # over and over again should not cause different results for [contour, numres] in tests: - assert len(simple_backtest(default_conf, contour, mocker, testdatadir)) == numres + assert len(simple_backtest(default_conf, contour, mocker, testdatadir)['results']) == numres @pytest.mark.parametrize('protections,contour,expected', [ @@ -589,10 +656,11 @@ def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir, default_conf['protections'] = protections default_conf['enable_protections'] = True + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) # While buy-signals are unrealistic, running backtesting # over and over again should not cause different results - assert len(simple_backtest(default_conf, contour, mocker, testdatadir)) == expected + assert len(simple_backtest(default_conf, contour, mocker, testdatadir)['results']) == expected def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): @@ -604,10 +672,11 @@ def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): backtest_conf = _make_backtest_conf(mocker, conf=default_conf, datadir=testdatadir) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override - results = backtesting.backtest(**backtest_conf) - assert results.empty + result = backtesting.backtest(**backtest_conf) + assert result['results'].empty def test_backtest_only_sell(mocker, default_conf, testdatadir): @@ -619,24 +688,28 @@ def test_backtest_only_sell(mocker, default_conf, testdatadir): backtest_conf = _make_backtest_conf(mocker, conf=default_conf, datadir=testdatadir) backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override - results = backtesting.backtest(**backtest_conf) - assert results.empty + result = backtesting.backtest(**backtest_conf) + assert result['results'].empty def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC', datadir=testdatadir) default_conf['timeframe'] = '1m' backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = _trend_alternate # Override backtesting.strategy.advise_sell = _trend_alternate # Override - results = backtesting.backtest(**backtest_conf) + result = backtesting.backtest(**backtest_conf) # 200 candles in backtest data # won't buy on first (shifted by 1) # 100 buys signals + results = result['results'] assert len(results) == 100 # One trade was force-closed at the end assert len(results.loc[results['is_open']]) == 0 @@ -658,6 +731,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) dataframe['sell'] = np.where((dataframe.index + multi - 2) % multi == 0, 1, 0) return dataframe + mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=0.00001) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) @@ -671,6 +745,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) default_conf['timeframe'] = '5m' backtesting = Backtesting(default_conf) + backtesting._set_strategy(backtesting.strategylist[0]) backtesting.strategy.advise_buy = _trend_alternate_hold # Override backtesting.strategy.advise_sell = _trend_alternate_hold # Override @@ -678,7 +753,6 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) min_date, max_date = get_timerange(processed) backtest_conf = { 'processed': processed, - 'stake_amount': default_conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, 'max_open_trades': 3, @@ -688,20 +762,19 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) results = backtesting.backtest(**backtest_conf) # Make sure we have parallel trades - assert len(evaluate_result_multi(results, '5m', 2)) > 0 + assert len(evaluate_result_multi(results['results'], '5m', 2)) > 0 # make sure we don't have trades with more than configured max_open_trades - assert len(evaluate_result_multi(results, '5m', 3)) == 0 + assert len(evaluate_result_multi(results['results'], '5m', 3)) == 0 backtest_conf = { 'processed': processed, - 'stake_amount': default_conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, 'max_open_trades': 1, 'position_stacking': False, } results = backtesting.backtest(**backtest_conf) - assert len(evaluate_result_multi(results, '5m', 1)) == 0 + assert len(evaluate_result_multi(results['results'], '5m', 1)) == 0 def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): @@ -733,9 +806,9 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Loading data from 2017-11-14 20:57:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...' ] @@ -746,8 +819,20 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): @pytest.mark.filterwarnings("ignore:deprecated") def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): + default_conf['ask_strategy'].update({ + "use_sell_signal": True, + "sell_profit_only": False, + "sell_profit_offset": 0.0, + "ignore_roi_if_buy_signal": False, + }) patch_exchange(mocker) - backtestmock = MagicMock(return_value=pd.DataFrame(columns=BT_DATA_COLUMNS)) + backtestmock = MagicMock(return_value={ + 'results': pd.DataFrame(columns=BT_DATA_COLUMNS), + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'final_balance': 1000, + }) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) @@ -761,7 +846,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): text_table_strategy=strattable_mock, generate_pair_metrics=MagicMock(), generate_sell_reason_stats=sell_reason_mock, - generate_strategy_metrics=strat_summary, + generate_strategy_comparison=strat_summary, generate_daily_stats=MagicMock(), ) patched_configuration_load_config_file(mocker, default_conf) @@ -795,9 +880,9 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Loading data from 2017-11-14 20:57:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy TestStrategyLegacy', @@ -809,39 +894,60 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): @pytest.mark.filterwarnings("ignore:deprecated") def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdatadir, capsys): - + default_conf['ask_strategy'].update({ + "use_sell_signal": True, + "sell_profit_only": False, + "sell_profit_offset": 0.0, + "ignore_roi_if_buy_signal": False, + }) patch_exchange(mocker) + result1 = pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC'], + 'profit_ratio': [0.0, 0.0], + 'profit_abs': [0.0, 0.0], + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', + '2018-01-30 03:30:00', ], utc=True + ), + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', + '2018-01-30 05:35:00', ], utc=True), + 'trade_duration': [235, 40], + 'is_open': [False, False], + 'stake_amount': [0.01, 0.01], + 'open_rate': [0.104445, 0.10302485], + 'close_rate': [0.104969, 0.103541], + 'sell_reason': [SellType.ROI, SellType.ROI] + }) + result2 = pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC', 'ETH/BTC'], + 'profit_ratio': [0.03, 0.01, 0.1], + 'profit_abs': [0.01, 0.02, 0.2], + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', + '2018-01-30 03:30:00', + '2018-01-30 05:30:00'], utc=True + ), + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', + '2018-01-30 05:35:00', + '2018-01-30 08:30:00'], utc=True), + 'trade_duration': [47, 40, 20], + 'is_open': [False, False, False], + 'stake_amount': [0.01, 0.01, 0.01], + 'open_rate': [0.104445, 0.10302485, 0.122541], + 'close_rate': [0.104969, 0.103541, 0.123541], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + }) backtestmock = MagicMock(side_effect=[ - pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC'], - 'profit_ratio': [0.0, 0.0], - 'profit_abs': [0.0, 0.0], - 'open_date': pd.to_datetime(['2018-01-29 18:40:00', - '2018-01-30 03:30:00', ], utc=True - ), - 'close_date': pd.to_datetime(['2018-01-29 20:45:00', - '2018-01-30 05:35:00', ], utc=True), - 'trade_duration': [235, 40], - 'is_open': [False, False], - 'open_rate': [0.104445, 0.10302485], - 'close_rate': [0.104969, 0.103541], - 'sell_reason': [SellType.ROI, SellType.ROI] - }), - pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC', 'ETH/BTC'], - 'profit_ratio': [0.03, 0.01, 0.1], - 'profit_abs': [0.01, 0.02, 0.2], - 'open_date': pd.to_datetime(['2018-01-29 18:40:00', - '2018-01-30 03:30:00', - '2018-01-30 05:30:00'], utc=True - ), - 'close_date': pd.to_datetime(['2018-01-29 20:45:00', - '2018-01-30 05:35:00', - '2018-01-30 08:30:00'], utc=True), - 'trade_duration': [47, 40, 20], - 'is_open': [False, False, False], - 'open_rate': [0.104445, 0.10302485, 0.122541], - 'close_rate': [0.104969, 0.103541, 0.123541], - 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] - }), + { + 'results': result1, + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'final_balance': 1000, + }, + { + 'results': result2, + 'config': default_conf, + 'locks': [], + 'rejected_signals': 20, + 'final_balance': 1000, + } ]) mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) @@ -872,9 +978,9 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Loading data from 2017-11-14 20:57:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Backtesting with data from 2017-11-14 21:17:00 ' - 'up to 2017-11-14 22:58:00 (0 days)..', + 'up to 2017-11-14 22:58:00 (0 days).', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy TestStrategyLegacy', diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index 188b4aa5f..6818a573b 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -4,8 +4,8 @@ from unittest.mock import MagicMock from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge +from freqtrade.enums import RunMode from freqtrade.optimize.edge_cli import EdgeCli -from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 68eb3d6f7..10e99395d 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,24 +1,27 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import locale import logging import re from datetime import datetime from pathlib import Path from typing import Dict, List -from unittest.mock import MagicMock +from unittest.mock import ANY, MagicMock import pandas as pd import pytest from arrow import Arrow from filelock import Timeout -from freqtrade import constants from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt from freqtrade.data.history import load_data -from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.enums import RunMode, SellType +from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt import Hyperopt +from freqtrade.optimize.hyperopt_auto import HyperOptAuto +from freqtrade.optimize.hyperopt_tools import HyperoptTools +from freqtrade.optimize.optimize_reports import generate_strategy_stats +from freqtrade.optimize.space import SKDecimal from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver -from freqtrade.state import RunMode +from freqtrade.strategy.hyper import IntParameter from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -26,23 +29,7 @@ from .hyperopts.default_hyperopt import DefaultHyperOpt # Functions for recurrent object patching -def create_results(mocker, hyperopt, testdatadir) -> List[Dict]: - """ - When creating results, mock the hyperopt so that *by default* - - we don't create any pickle'd files in the filesystem - - we might have a pickle'd file so make sure that we return - false when looking for it - """ - hyperopt.results_file = testdatadir / 'optimize/ut_results.pickle' - - mocker.patch.object(Path, "is_file", MagicMock(return_value=False)) - stat_mock = MagicMock() - stat_mock.st_size = 1 - mocker.patch.object(Path, "stat", MagicMock(return_value=stat_mock)) - - mocker.patch.object(Path, "unlink", MagicMock(return_value=True)) - mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) - mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') +def create_results() -> List[Dict]: return [{'loss': 1, 'result': 'foo', 'params': {}, 'is_best': True}] @@ -130,8 +117,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo assert log_has('Parameter --print-all detected ...', caplog) -def test_setup_hyperopt_configuration_unlimited_stake_amount(mocker, default_conf) -> None: - default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT +def test_setup_hyperopt_configuration_stake_amount(mocker, default_conf) -> None: patched_configuration_load_config_file(mocker, default_conf) @@ -139,9 +125,20 @@ def test_setup_hyperopt_configuration_unlimited_stake_amount(mocker, default_con 'hyperopt', '--config', 'config.json', '--hyperopt', 'DefaultHyperOpt', + '--stake-amount', '1', + '--starting-balance', '2' ] + conf = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) + assert isinstance(conf, dict) - with pytest.raises(DependencyException, match=r'.`stake_amount`.*'): + args = [ + 'hyperopt', + '--config', 'config.json', + '--strategy', 'DefaultStrategy', + '--stake-amount', '1', + '--starting-balance', '0.5' + ] + with pytest.raises(OperationalException, match=r"Starting balance .* smaller .*"): setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) @@ -306,55 +303,50 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_save_results_saves_epochs(mocker, hyperopt, testdatadir, caplog) -> None: - epochs = create_results(mocker, hyperopt, testdatadir) - mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) - mock_dump_json = mocker.patch('freqtrade.optimize.hyperopt.file_dump_json', return_value=None) - results_file = testdatadir / 'optimize' / 'ut_results.pickle' +def test_save_results_saves_epochs(mocker, hyperopt, tmpdir, caplog) -> None: + # Test writing to temp dir and reading again + epochs = create_results() + hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt') caplog.set_level(logging.DEBUG) - hyperopt.epochs = epochs - hyperopt._save_results() - assert log_has(f"1 epoch saved to '{results_file}'.", caplog) - mock_dump.assert_called_once() - mock_dump_json.assert_called_once() + for epoch in epochs: + hyperopt._save_result(epoch) + assert log_has(f"1 epoch saved to '{hyperopt.results_file}'.", caplog) - hyperopt.epochs = epochs + epochs - hyperopt._save_results() - assert log_has(f"2 epochs saved to '{results_file}'.", caplog) + hyperopt._save_result(epochs[0]) + assert log_has(f"2 epochs saved to '{hyperopt.results_file}'.", caplog) + + hyperopt_epochs = HyperoptTools.load_previous_results(hyperopt.results_file) + assert len(hyperopt_epochs) == 2 -def test_read_results_returns_epochs(mocker, hyperopt, testdatadir, caplog) -> None: - epochs = create_results(mocker, hyperopt, testdatadir) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=epochs) - results_file = testdatadir / 'optimize' / 'ut_results.pickle' - hyperopt_epochs = hyperopt._read_results(results_file) - assert log_has(f"Reading epochs from '{results_file}'", caplog) - assert hyperopt_epochs == epochs - mock_load.assert_called_once() +def test_load_previous_results(testdatadir, caplog) -> None: + + results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' + + hyperopt_epochs = HyperoptTools.load_previous_results(results_file) + + assert len(hyperopt_epochs) == 5 + assert log_has_re(r"Reading pickled epochs from .*", caplog) + + caplog.clear() + + # Modern version + results_file = testdatadir / 'strategy_SampleStrategy.fthypt' + + hyperopt_epochs = HyperoptTools.load_previous_results(results_file) + + assert len(hyperopt_epochs) == 5 + assert log_has_re(r"Reading epochs from .*", caplog) -def test_load_previous_results(mocker, hyperopt, testdatadir, caplog) -> None: - epochs = create_results(mocker, hyperopt, testdatadir) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=epochs) - mocker.patch.object(Path, 'is_file', MagicMock(return_value=True)) - statmock = MagicMock() - statmock.st_size = 5 - # mocker.patch.object(Path, 'stat', MagicMock(return_value=statmock)) - - results_file = testdatadir / 'optimize' / 'ut_results.pickle' - - hyperopt_epochs = hyperopt.load_previous_results(results_file) - - assert hyperopt_epochs == epochs - mock_load.assert_called_once() - - del epochs[0]['is_best'] - mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=epochs) - - with pytest.raises(OperationalException): - hyperopt.load_previous_results(results_file) +def test_load_previous_results2(mocker, testdatadir, caplog) -> None: + mocker.patch('freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results_pickle', + return_value=[{'asdf': '222'}]) + results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' + with pytest.raises(OperationalException, match=r"The file .* incompatible.*"): + HyperoptTools.load_previous_results(results_file) def test_roi_table_generation(hyperopt) -> None: @@ -371,7 +363,8 @@ def test_roi_table_generation(hyperopt) -> None: def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: - dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump') + dumper2 = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._save_result') mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', @@ -410,9 +403,9 @@ def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out - assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called for historical candle data + assert dumper.call_count == 1 + assert dumper2.call_count == 1 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") @@ -420,18 +413,42 @@ def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: assert hasattr(hyperopt, "position_stacking") -def test_format_results(hyperopt): - # Test with BTC as stake_currency - trades = [ - ('ETH/BTC', 2, 2, 123), - ('LTC/BTC', 1, 1, 123), - ('XPR/BTC', -1, -2, -246) - ] - labels = ['currency', 'profit_ratio', 'profit_abs', 'trade_duration'] - df = pd.DataFrame.from_records(trades, columns=labels) - results_metrics = hyperopt._calculate_results_metrics(df) - results_explanation = hyperopt._format_results_explanation_string(results_metrics) - total_profit = results_metrics['total_profit'] +def test_hyperopt_format_results(hyperopt): + + bt_result = { + 'results': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", + "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], + "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], + "trade_duration": [123, 34, 31, 14], + "is_open": [False, False, False, True], + "stake_amount": [0.01, 0.01, 0.01, 0.01], + "sell_reason": [SellType.ROI, SellType.STOP_LOSS, + SellType.ROI, SellType.FORCE_SELL] + }), + 'config': hyperopt.config, + 'locks': [], + 'final_balance': 0.02, + 'rejected_signals': 2, + 'backtest_start_time': 1619718665, + 'backtest_end_time': 1619718665, + } + results_metrics = generate_strategy_stats({'XRP/BTC': None}, '', bt_result, + Arrow(2017, 11, 14, 19, 32, 00), + Arrow(2017, 12, 14, 19, 32, 00), market_change=0) + + results_explanation = HyperoptTools.format_results_explanation_string(results_metrics, 'BTC') + total_profit = results_metrics['profit_total_abs'] results = { 'loss': 0.0, @@ -444,22 +461,10 @@ def test_format_results(hyperopt): 'is_initial_point': True, } - result = hyperopt._format_explanation_string(results, 1) - assert result.find(' 66.67%') - assert result.find('Total profit 1.00000000 BTC') - assert result.find('2.0000Σ %') - - # Test with EUR as stake_currency - trades = [ - ('ETH/EUR', 2, 2, 123), - ('LTC/EUR', 1, 1, 123), - ('XPR/EUR', -1, -2, -246) - ] - df = pd.DataFrame.from_records(trades, columns=labels) - results_metrics = hyperopt._calculate_results_metrics(df) - results['total_profit'] = results_metrics['total_profit'] - result = hyperopt._format_explanation_string(results, 1) - assert result.find('Total profit 1.00000000 EUR') + result = HyperoptTools._format_explanation_string(results, 1) + assert ' 0.71%' in result + assert 'Total profit 0.00003100 BTC' in result + assert '0:50:00 min' in result @pytest.mark.parametrize("spaces, expected_results", [ @@ -490,10 +495,10 @@ def test_format_results(hyperopt): (['default', 'buy'], {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), ]) -def test_has_space(hyperopt, spaces, expected_results): +def test_has_space(hyperopt_conf, spaces, expected_results): for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: - hyperopt.config.update({'spaces': spaces}) - assert hyperopt.has_space(s) == expected_results[s] + hyperopt_conf.update({'spaces': spaces}) + assert HyperoptTools.has_space(hyperopt_conf, s) == expected_results[s] def test_populate_indicators(hyperopt, testdatadir) -> None: @@ -564,22 +569,39 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'hyperopt_min_trades': 1, }) - trades = [ - ('TRX/BTC', 0.023117, 0.000233, 100) - ] - labels = ['currency', 'profit_ratio', 'profit_abs', 'trade_duration'] - backtest_result = pd.DataFrame.from_records(trades, columns=labels) + backtest_result = { + 'results': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", + "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_ratio": [0.003312, 0.010801, 0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], + "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], + "trade_duration": [123, 34, 31, 14], + "is_open": [False, False, False, True], + "stake_amount": [0.01, 0.01, 0.01, 0.01], + "sell_reason": [SellType.ROI, SellType.STOP_LOSS, + SellType.ROI, SellType.FORCE_SELL] + }), + 'config': hyperopt_conf, + 'locks': [], + 'rejected_signals': 20, + 'final_balance': 1000, + } - mocker.patch( - 'freqtrade.optimize.hyperopt.Backtesting.backtest', - MagicMock(return_value=backtest_result) - ) - mocker.patch( - 'freqtrade.optimize.hyperopt.get_timerange', - MagicMock(return_value=(Arrow(2017, 12, 10), Arrow(2017, 12, 13))) - ) + mocker.patch('freqtrade.optimize.hyperopt.Backtesting.backtest', return_value=backtest_result) + mocker.patch('freqtrade.optimize.hyperopt.get_timerange', + return_value=(Arrow(2017, 12, 10), Arrow(2017, 12, 13))) patch_exchange(mocker) - mocker.patch('freqtrade.optimize.hyperopt.load', MagicMock()) + mocker.patch.object(Path, 'open') + mocker.patch('freqtrade.optimize.hyperopt.load', return_value={'XRP/BTC': None}) optimizer_param = { 'adx-value': 0, @@ -613,12 +635,12 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'trailing_only_offset_is_reached': False, } response_expected = { - 'loss': 1.9840569076926293, - 'results_explanation': (' 1 trades. 1/0/0 Wins/Draws/Losses. ' - 'Avg profit 2.31%. Median profit 2.31%. Total profit ' - '0.00023300 BTC ( 2.31\N{GREEK CAPITAL LETTER SIGMA}%). ' - 'Avg duration 100.0 min.' - ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8'), + 'loss': 1.9147239021396234, + 'results_explanation': (' 4 trades. 4/0/0 Wins/Draws/Losses. ' + 'Avg profit 0.77%. Median profit 0.71%. Total profit ' + '0.00003100 BTC ( 0.00%). ' + 'Avg duration 0:50:00 min.' + ), 'params_details': {'buy': {'adx-enabled': False, 'adx-value': 0, 'fastd-enabled': True, @@ -628,10 +650,10 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'rsi-enabled': False, 'rsi-value': 0, 'trigger': 'macd_cross_signal'}, - 'roi': {0: 0.12000000000000001, - 20.0: 0.02, - 50.0: 0.01, - 110.0: 0}, + 'roi': {"0": 0.12000000000000001, + "20.0": 0.02, + "50.0": 0.01, + "110.0": 0}, 'sell': {'sell-adx-enabled': False, 'sell-adx-value': 0, 'sell-fastd-enabled': True, @@ -647,21 +669,16 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'trailing_stop_positive': 0.02, 'trailing_stop_positive_offset': 0.07}}, 'params_dict': optimizer_param, - 'results_metrics': {'avg_profit': 2.3117, - 'draws': 0, - 'duration': 100.0, - 'losses': 0, - 'winsdrawslosses': ' 1 0 0', - 'median_profit': 2.3117, - 'profit': 2.3117, - 'total_profit': 0.000233, - 'trade_count': 1, - 'wins': 1}, - 'total_profit': 0.00023300 + 'params_not_optimized': {'buy': {}, 'sell': {}}, + 'results_metrics': ANY, + 'total_profit': 3.1e-08 } hyperopt = Hyperopt(hyperopt_conf) - hyperopt.dimensions = hyperopt.hyperopt_space() + hyperopt.min_date = Arrow(2017, 12, 10) + hyperopt.max_date = Arrow(2017, 12, 13) + hyperopt.init_spaces() + hyperopt.dimensions = hyperopt.dimensions generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) assert generate_optimizer_value == response_expected @@ -678,7 +695,8 @@ def test_clean_hyperopt(mocker, hyperopt_conf, caplog): def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: - dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump') + dumper2 = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._save_result') mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', @@ -729,13 +747,14 @@ def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: ':{},"stoploss":null,"trailing_stop":null}' ) assert result_str in out # noqa: E501 - assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called for historical candle data + assert dumper.call_count == 1 + assert dumper2.call_count == 1 def test_print_json_spaces_default(mocker, hyperopt_conf, capsys) -> None: - dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump') + dumper2 = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._save_result') mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) @@ -777,13 +796,14 @@ def test_print_json_spaces_default(mocker, hyperopt_conf, capsys) -> None: out, err = capsys.readouterr() assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null}' in out # noqa: E501 - assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + # Should be called for historical candle data + assert dumper.call_count == 1 + assert dumper2.call_count == 1 def test_print_json_spaces_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: - dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump') + dumper2 = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._save_result') mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) @@ -824,13 +844,14 @@ def test_print_json_spaces_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: out, err = capsys.readouterr() assert '{"minimal_roi":{},"stoploss":null}' in out - assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + + assert dumper.call_count == 1 + assert dumper2.call_count == 1 def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: - dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump') + dumper2 = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._save_result') mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) @@ -872,9 +893,9 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out - assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + assert dumper.call_count == 1 + assert dumper2.call_count == 1 + assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") @@ -910,7 +931,8 @@ def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None: def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: - dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump') + dumper2 = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._save_result') mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) @@ -953,8 +975,8 @@ def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + assert dumper.call_count == 1 + assert dumper2.call_count == 1 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") @@ -963,7 +985,8 @@ def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: - dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + dumper = mocker.patch('freqtrade.optimize.hyperopt.dump') + dumper2 = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._save_result') mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) @@ -1006,8 +1029,8 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for historical candle data, once to save evaluations - assert dumper.call_count == 2 + assert dumper.call_count == 1 + assert dumper2.call_count == 1 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") @@ -1045,7 +1068,7 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No hyperopt.start() -def test_print_epoch_details(capsys): +def test_show_epoch_details(capsys): test_result = { 'params_details': { 'trailing': { @@ -1067,7 +1090,7 @@ def test_print_epoch_details(capsys): 'is_best': True } - Hyperopt.print_epoch_details(test_result, 5, False, no_header=True) + HyperoptTools.show_epoch_details(test_result, 5, False, no_header=True) captured = capsys.readouterr() assert '# Trailing stop:' in captured.out # re.match(r"Pairs for .*", captured.out) @@ -1079,3 +1102,58 @@ def test_print_epoch_details(capsys): assert '# ROI table:' in captured.out assert re.search(r'^\s+minimal_roi = \{$', captured.out, re.MULTILINE) assert re.search(r'^\s+\"90\"\:\s0.14,\s*$', captured.out, re.MULTILINE) + + +def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True) + # No hyperopt needed + del hyperopt_conf['hyperopt'] + hyperopt_conf.update({ + 'strategy': 'HyperoptableStrategy', + 'user_data_dir': Path(tmpdir), + }) + hyperopt = Hyperopt(hyperopt_conf) + assert isinstance(hyperopt.custom_hyperopt, HyperOptAuto) + assert isinstance(hyperopt.backtesting.strategy.buy_rsi, IntParameter) + + assert hyperopt.backtesting.strategy.buy_rsi.in_space is True + assert hyperopt.backtesting.strategy.buy_rsi.value == 35 + buy_rsi_range = hyperopt.backtesting.strategy.buy_rsi.range + assert isinstance(buy_rsi_range, range) + # Range from 0 - 50 (inclusive) + assert len(list(buy_rsi_range)) == 51 + + hyperopt.start() + + +def test_SKDecimal(): + space = SKDecimal(1, 2, decimals=2) + assert 1.5 in space + assert 2.5 not in space + assert space.low == 100 + assert space.high == 200 + + assert space.inverse_transform([200]) == [2.0] + assert space.inverse_transform([100]) == [1.0] + assert space.inverse_transform([150, 160]) == [1.5, 1.6] + + assert space.transform([1.5]) == [150] + assert space.transform([2.0]) == [200] + assert space.transform([1.0]) == [100] + assert space.transform([1.5, 1.6]) == [150, 160] + + +def test___pprint(): + params = {'buy_std': 1.2, 'buy_rsi': 31, 'buy_enable': True, 'buy_what': 'asdf'} + non_params = {'buy_notoptimied': 55} + + x = HyperoptTools._pprint(params, non_params) + assert x == """{ + "buy_std": 1.2, + "buy_rsi": 31, + "buy_enable": True, + "buy_what": "asdf", + "buy_notoptimied": 55, # value loaded from strategy +}""" diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index 73feeb007..ea0caac04 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -149,9 +149,9 @@ def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_result def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: results_over = hyperopt_results.copy() - results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 + results_over['profit_abs'] = hyperopt_results['profit_abs'] * 2 results_under = hyperopt_results.copy() - results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 + results_under['profit_abs'] = hyperopt_results['profit_abs'] / 2 default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) hl = HyperOptLossResolver.load_hyperoptloss(default_conf) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 51a78c7cc..3f31efb95 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -1,5 +1,6 @@ +import datetime import re -from datetime import datetime, timedelta, timezone +from datetime import timedelta from pathlib import Path import pandas as pd @@ -7,18 +8,19 @@ import pytest from arrow import Arrow from freqtrade.configuration import TimeRange -from freqtrade.constants import LAST_BT_RESULT_FN +from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN from freqtrade.data import history from freqtrade.data.btanalysis import get_latest_backtest_filename, load_backtest_data from freqtrade.edge import PairInfo +from freqtrade.enums import SellType from freqtrade.optimize.optimize_reports import (generate_backtest_stats, generate_daily_stats, generate_edge_table, generate_pair_metrics, generate_sell_reason_stats, - generate_strategy_metrics, store_backtest_stats, + generate_strategy_comparison, + generate_trading_stats, store_backtest_stats, text_table_bt_results, text_table_sell_reason, text_table_strategy) from freqtrade.resolvers.strategy_resolver import StrategyResolver -from freqtrade.strategy.interface import SellType from tests.data.test_history import _backup_file, _clean_test_file @@ -26,33 +28,30 @@ def test_text_table_bt_results(): results = pd.DataFrame( { - 'pair': ['ETH/BTC', 'ETH/BTC'], - 'profit_ratio': [0.1, 0.2], - 'profit_abs': [0.2, 0.4], - 'trade_duration': [10, 30], - 'wins': [2, 0], - 'draws': [0, 0], - 'losses': [0, 0] + 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'profit_ratio': [0.1, 0.2, -0.05], + 'profit_abs': [0.2, 0.4, -0.1], + 'trade_duration': [10, 30, 20], } ) result_str = ( - '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' - ' Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' - '|---------+--------+----------------+----------------+------------------+' - '----------------+----------------+--------+---------+----------|\n' - '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 |' - ' 15.00 | 0:20:00 | 2 | 0 | 0 |\n' - '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 |' - ' 15.00 | 0:20:00 | 2 | 0 | 0 |' + '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |' + ' Avg Duration | Win Draw Loss Win% |\n' + '|---------+--------+----------------+----------------+------------------+----------------+' + '----------------+-------------------------|\n' + '| ETH/BTC | 3 | 8.33 | 25.00 | 0.50000000 | 12.50 |' + ' 0:20:00 | 2 0 1 66.7 |\n' + '| TOTAL | 3 | 8.33 | 25.00 | 0.50000000 | 12.50 |' + ' 0:20:00 | 2 0 1 66.7 |' ) pair_results = generate_pair_metrics(data={'ETH/BTC': {}}, stake_currency='BTC', - max_open_trades=2, results=results) + starting_balance=4, results=results) assert text_table_bt_results(pair_results, stake_currency='BTC') == result_str -def test_generate_backtest_stats(default_conf, testdatadir): +def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): default_conf.update({'strategy': 'DefaultStrategy'}) StrategyResolver.load_strategy(default_conf) @@ -73,11 +72,14 @@ def test_generate_backtest_stats(default_conf, testdatadir): "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], "trade_duration": [123, 34, 31, 14], "is_open": [False, False, False, True], + "stake_amount": [0.01, 0.01, 0.01, 0.01], "sell_reason": [SellType.ROI, SellType.STOP_LOSS, SellType.ROI, SellType.FORCE_SELL] }), 'config': default_conf, 'locks': [], + 'final_balance': 1000.02, + 'rejected_signals': 20, 'backtest_start_time': Arrow.utcnow().int_timestamp, 'backtest_end_time': Arrow.utcnow().int_timestamp, } @@ -94,12 +96,13 @@ def test_generate_backtest_stats(default_conf, testdatadir): assert 'DefStrat' in stats['strategy'] assert 'strategy_comparison' in stats strat_stats = stats['strategy']['DefStrat'] - assert strat_stats['backtest_start'] == min_date.datetime - assert strat_stats['backtest_end'] == max_date.datetime + assert strat_stats['backtest_start'] == min_date.strftime(DATETIME_PRINT_FORMAT) + assert strat_stats['backtest_end'] == max_date.strftime(DATETIME_PRINT_FORMAT) assert strat_stats['total_trades'] == len(results['DefStrat']['results']) # Above sample had no loosing trade assert strat_stats['max_drawdown'] == 0.0 + # Retry with losing trade results = {'DefStrat': { 'results': pd.DataFrame( {"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"], @@ -116,23 +119,37 @@ def test_generate_backtest_stats(default_conf, testdatadir): "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217], "trade_duration": [123, 34, 31, 14], - "open_at_end": [False, False, False, True], - "sell_reason": [SellType.ROI, SellType.STOP_LOSS, - SellType.ROI, SellType.FORCE_SELL] + "is_open": [False, False, False, True], + "stake_amount": [0.01, 0.01, 0.01, 0.01], + "sell_reason": [SellType.ROI, SellType.ROI, + SellType.STOP_LOSS, SellType.FORCE_SELL] }), - 'config': default_conf} + 'config': default_conf, + 'locks': [], + 'final_balance': 1000.02, + 'rejected_signals': 20, + 'backtest_start_time': Arrow.utcnow().int_timestamp, + 'backtest_end_time': Arrow.utcnow().int_timestamp, + } } - assert strat_stats['max_drawdown'] == 0.0 - assert strat_stats['drawdown_start'] == datetime(1970, 1, 1, tzinfo=timezone.utc) - assert strat_stats['drawdown_end'] == datetime(1970, 1, 1, tzinfo=timezone.utc) - assert strat_stats['drawdown_end_ts'] == 0 - assert strat_stats['drawdown_start_ts'] == 0 + stats = generate_backtest_stats(btdata, results, min_date, max_date) + assert isinstance(stats, dict) + assert 'strategy' in stats + assert 'DefStrat' in stats['strategy'] + assert 'strategy_comparison' in stats + strat_stats = stats['strategy']['DefStrat'] + + assert strat_stats['max_drawdown'] == 0.013803 + assert strat_stats['drawdown_start'] == '2017-11-14 22:10:00' + assert strat_stats['drawdown_end'] == '2017-11-14 22:43:00' + assert strat_stats['drawdown_end_ts'] == 1510699380000 + assert strat_stats['drawdown_start_ts'] == 1510697400000 assert strat_stats['pairlist'] == ['UNITTEST/BTC'] # Test storing stats - filename = Path(testdatadir / 'btresult.json') - filename_last = Path(testdatadir / LAST_BT_RESULT_FN) + filename = Path(tmpdir / 'btresult.json') + filename_last = Path(tmpdir / LAST_BT_RESULT_FN) _backup_file(filename_last, copy_file=True) assert not filename.is_file() @@ -142,7 +159,7 @@ def test_generate_backtest_stats(default_conf, testdatadir): last_fn = get_latest_backtest_filename(filename_last.parent) assert re.match(r"btresult-.*\.json", last_fn) - filename1 = (testdatadir / last_fn) + filename1 = Path(tmpdir / last_fn) assert filename1.is_file() content = filename1.read_text() assert 'max_drawdown' in content @@ -189,7 +206,7 @@ def test_generate_pair_metrics(): ) pair_results = generate_pair_metrics(data={'ETH/BTC': {}}, stake_currency='BTC', - max_open_trades=2, results=results) + starting_balance=2, results=results) assert isinstance(pair_results, list) assert len(pair_results) == 2 assert pair_results[-1]['key'] == 'TOTAL' @@ -210,8 +227,6 @@ def test_generate_daily_stats(testdatadir): assert res['winning_days'] == 14 assert res['draw_days'] == 4 assert res['losing_days'] == 3 - assert res['winner_holding_avg'] == timedelta(seconds=1440) - assert res['loser_holding_avg'] == timedelta(days=1, seconds=21420) # Select empty dataframe! res = generate_daily_stats(bt_data.loc[bt_data['open_date'] == '2000-01-01', :]) @@ -222,6 +237,23 @@ def test_generate_daily_stats(testdatadir): assert res['losing_days'] == 0 +def test_generate_trading_stats(testdatadir): + filename = testdatadir / "backtest-result_new.json" + bt_data = load_backtest_data(filename) + res = generate_trading_stats(bt_data) + assert isinstance(res, dict) + assert res['winner_holding_avg'] == timedelta(seconds=1440) + assert res['loser_holding_avg'] == timedelta(days=1, seconds=21420) + assert 'wins' in res + assert 'losses' in res + assert 'draws' in res + + # Select empty dataframe! + res = generate_trading_stats(bt_data.loc[bt_data['open_date'] == '2000-01-01', :]) + assert res['wins'] == 0 + assert res['losses'] == 0 + + def test_text_table_sell_reason(): results = pd.DataFrame( @@ -238,14 +270,14 @@ def test_text_table_sell_reason(): ) result_str = ( - '| Sell Reason | Sells | Wins | Draws | Losses |' - ' Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |\n' - '|---------------+---------+--------+---------+----------+' - '----------------+----------------+------------------+----------------|\n' - '| roi | 2 | 2 | 0 | 0 |' - ' 15 | 30 | 0.6 | 15 |\n' - '| stop_loss | 1 | 0 | 0 | 1 |' - ' -10 | -10 | -0.2 | -5 |' + '| Sell Reason | Sells | Win Draws Loss Win% | Avg Profit % | Cum Profit % |' + ' Tot Profit BTC | Tot Profit % |\n' + '|---------------+---------+--------------------------+----------------+----------------+' + '------------------+----------------|\n' + '| roi | 2 | 2 0 0 100 | 15 | 30 |' + ' 0.6 | 15 |\n' + '| stop_loss | 1 | 0 0 1 0 | -10 | -10 |' + ' -0.2 | -5 |' ) sell_reason_stats = generate_sell_reason_stats(max_open_trades=2, @@ -265,7 +297,7 @@ def test_generate_sell_reason_stats(): 'wins': [2, 0, 0], 'draws': [0, 0, 0], 'losses': [0, 0, 1], - 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + 'sell_reason': [SellType.ROI.value, SellType.ROI.value, SellType.STOP_LOSS.value] } ) @@ -291,10 +323,14 @@ def test_generate_sell_reason_stats(): def test_text_table_strategy(default_conf): default_conf['max_open_trades'] = 2 + default_conf['dry_run_wallet'] = 3 results = {} + date = datetime.datetime(year=2020, month=1, day=1, hour=12, minute=30) + delta = datetime.timedelta(days=1) results['TestStrategy1'] = {'results': pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'close_date': [date, date + delta, date + delta * 2], 'profit_ratio': [0.1, 0.2, 0.3], 'profit_abs': [0.2, 0.4, 0.5], 'trade_duration': [10, 30, 10], @@ -307,6 +343,7 @@ def test_text_table_strategy(default_conf): results['TestStrategy2'] = {'results': pd.DataFrame( { 'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'], + 'close_date': [date, date + delta, date + delta * 2], 'profit_ratio': [0.4, 0.2, 0.3], 'profit_abs': [0.4, 0.4, 0.5], 'trade_duration': [15, 30, 15], @@ -318,18 +355,17 @@ def test_text_table_strategy(default_conf): ), 'config': default_conf} result_str = ( - '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' - ' Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' + ' Tot Profit % | Avg Duration | Win Draw Loss Win% | Drawdown |\n' '|---------------+--------+----------------+----------------+------------------+' - '----------------+----------------+--------+---------+----------|\n' + '----------------+----------------+-------------------------+-----------------------|\n' '| TestStrategy1 | 3 | 20.00 | 60.00 | 1.10000000 |' - ' 30.00 | 0:17:00 | 3 | 0 | 0 |\n' + ' 36.67 | 0:17:00 | 3 0 0 100 | 0.00000000 BTC 0.00% |\n' '| TestStrategy2 | 3 | 30.00 | 90.00 | 1.30000000 |' - ' 45.00 | 0:20:00 | 3 | 0 | 0 |' + ' 43.33 | 0:20:00 | 3 0 0 100 | 0.00000000 BTC 0.00% |' ) - strategy_results = generate_strategy_metrics(all_results=results) - + strategy_results = generate_strategy_comparison(all_results=results) assert text_table_strategy(strategy_results, 'BTC') == result_str diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 67cd96f5b..5e2274ce3 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -1,15 +1,17 @@ # pragma pylint: disable=missing-docstring,C0103,protected-access +import time from unittest.mock import MagicMock, PropertyMock import pytest from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.exceptions import OperationalException +from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver -from tests.conftest import get_patched_freqtradebot, log_has, log_has_re +from tests.conftest import get_patched_exchange, get_patched_freqtradebot, log_has, log_has_re @pytest.fixture(scope="function") @@ -260,6 +262,8 @@ def test_refresh_pairlist_dynamic_2(mocker, shitcoinmarkets, tickers, whitelist_ freqtrade.pairlists.refresh_pairlist() assert whitelist == freqtrade.pairlists.whitelist + # Delay to allow 0 TTL cache to expire... + time.sleep(1) whitelist = ['FUEL/BTC', 'ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC'] tickers_dict['FUEL/BTC']['quoteVolume'] = 10000.0 freqtrade.pairlists.refresh_pairlist() @@ -403,10 +407,17 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.02}], "USDT", ['ETH/USDT', 'NANO/USDT']), + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "PriceFilter", "max_value": 0.000001}], + "USDT", ['NANO/USDT']), ([{"method": "StaticPairList"}, {"method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01, "refresh_period": 1440}], "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), + ([{"method": "StaticPairList"}, + {"method": "VolatilityFilter", "lookback_days": 3, + "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}], + "BTC", ['ETH/BTC', 'TKN/BTC']) ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history, pairlists, base_currency, @@ -414,12 +425,15 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency + ohlcv_history_high_vola = ohlcv_history.copy() + ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090 + ohlcv_data = { ('ETH/BTC', '1d'): ohlcv_history, ('TKN/BTC', '1d'): ohlcv_history, ('LTC/BTC', '1d'): ohlcv_history, ('XRP/BTC', '1d'): ohlcv_history, - ('HOT/BTC', '1d'): ohlcv_history, + ('HOT/BTC', '1d'): ohlcv_history_high_vola, } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -478,6 +492,8 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t r'because last price < .*%$', caplog) or log_has_re(r'^Removed .* from whitelist, ' r'because last price > .*%$', caplog) or + log_has_re(r'^Removed .* from whitelist, ' + r'because min value change of .*', caplog) or log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] " r"is empty.*", caplog)) if pairlist['method'] == 'VolumePairList': @@ -487,6 +503,8 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t assert log_has(logmsg, caplog) else: assert not log_has(logmsg, caplog) + if pairlist["method"] == 'VolatilityFilter': + assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog) def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: @@ -500,6 +518,18 @@ def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: PairListManager(MagicMock, whitelist_conf) +def test_PerformanceFilter_error(mocker, whitelist_conf, caplog) -> None: + whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}] + if hasattr(Trade, 'query'): + del Trade.query + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + exchange = get_patched_exchange(mocker, whitelist_conf) + pm = PairListManager(exchange, whitelist_conf) + pm.refresh_pairlist() + + assert log_has("PerformanceFilter is not available in this mode.", caplog) + + def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}] @@ -595,17 +625,14 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): get_tickers=tickers ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) - assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == 0 + assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 0 assert tickers.call_count == 0 freqtrade.pairlists.refresh_pairlist() assert tickers.call_count == 1 - assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh != 0 - lrf = freqtrade.pairlists._pairlist_handlers[0]._last_refresh + assert len(freqtrade.pairlists._pairlist_handlers[0]._pair_cache) == 1 freqtrade.pairlists.refresh_pairlist() assert tickers.call_count == 1 - # Time should not be updated. - assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers): @@ -778,6 +805,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.00002000.'}]", None ), + ({"method": "PriceFilter", "max_value": 0.00002000}, + "[{'PriceFilter': 'PriceFilter - Filtering pairs priced Value above 0.00002000.'}]", + None + ), ({"method": "PriceFilter"}, "[{'PriceFilter': 'PriceFilter - No price filters configured.'}]", None @@ -794,6 +825,10 @@ def test_spreadfilter_invalid_data(mocker, default_conf, markets, tickers, caplo None, "PriceFilter requires max_price to be >= 0" ), # OperationalException expected + ({"method": "PriceFilter", "max_value": -1.00010000}, + None, + "PriceFilter requires max_value to be >= 0" + ), # OperationalException expected ({"method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01}, "[{'RangeStabilityFilter': 'RangeStabilityFilter - Filtering pairs with rate of change below " "0.01 over the last days.'}]", diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index dfcbff0ed..fce3a8cd1 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -73,9 +73,13 @@ def test_PairLocks(use_db): assert PairLocks.is_pair_locked('XRP/USDT', lock_time + timedelta(minutes=-50)) if use_db: - assert len(PairLock.query.all()) > 0 + locks = PairLocks.get_all_locks() + locks_db = PairLock.query.all() + assert len(locks) == len(locks_db) + assert len(locks_db) > 0 else: # Nothing was pushed to the database + assert len(PairLocks.get_all_locks()) > 0 assert len(PairLock.query.all()) == 0 # Reset use-db variable PairLocks.reset_locks() diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index e36900a96..10ab64690 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -4,9 +4,9 @@ from datetime import datetime, timedelta import pytest from freqtrade import constants +from freqtrade.enums import SellType from freqtrade.persistence import PairLocks, Trade from freqtrade.plugins.protectionmanager import ProtectionManager -from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has_re @@ -27,7 +27,7 @@ def generate_mock_trade(pair: str, fee: float, is_open: bool, open_rate=open_rate, is_open=is_open, amount=0.01 / open_rate, - exchange='bittrex', + exchange='binance', ) trade.recalc_open_trade_value() if not is_open: @@ -83,7 +83,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): "method": "StoplossGuard", "lookback_period": 60, "stop_duration": 40, - "trade_limit": 2 + "trade_limit": 3 }] freqtrade = get_patched_freqtradebot(mocker, default_conf) message = r"Trading stopped due to .*" @@ -91,7 +91,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): assert not log_has_re(message, caplog) caplog.clear() - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, )) @@ -100,12 +100,12 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): assert not log_has_re(message, caplog) caplog.clear() # This trade does not count, as it's closed too long ago - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'BCH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=250, min_ago_close=100, )) - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=240, min_ago_close=30, )) @@ -114,7 +114,7 @@ def test_stoploss_guard(mocker, default_conf, fee, caplog): assert not log_has_re(message, caplog) caplog.clear() - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'LTC/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=180, min_ago_close=30, )) @@ -136,7 +136,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair default_conf['protections'] = [{ "method": "StoplossGuard", "lookback_period": 60, - "trade_limit": 1, + "trade_limit": 2, "stop_duration": 60, "only_per_pair": only_per_pair }] @@ -148,7 +148,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair assert not log_has_re(message, caplog) caplog.clear() - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, profit_rate=0.9, )) @@ -158,12 +158,12 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair assert not log_has_re(message, caplog) caplog.clear() # This trade does not count, as it's closed too long ago - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=250, min_ago_close=100, profit_rate=0.9, )) # Trade does not count for per pair stop as it's the wrong pair. - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=240, min_ago_close=30, profit_rate=0.9, )) @@ -178,7 +178,7 @@ def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair caplog.clear() # 2nd Trade that counts with correct pair - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=180, min_ago_close=30, profit_rate=0.9, )) @@ -203,7 +203,7 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog): assert not log_has_re(message, caplog) caplog.clear() - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=200, min_ago_close=30, )) @@ -213,7 +213,7 @@ def test_CooldownPeriod(mocker, default_conf, fee, caplog): assert PairLocks.is_pair_locked('XRP/BTC') assert not PairLocks.is_global_lock() - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'ETH/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, min_ago_open=205, min_ago_close=35, )) @@ -242,7 +242,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert not log_has_re(message, caplog) caplog.clear() - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=800, min_ago_close=450, profit_rate=0.9, )) @@ -253,7 +253,7 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert not PairLocks.is_pair_locked('XRP/BTC') assert not PairLocks.is_global_lock() - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=200, min_ago_close=120, profit_rate=0.9, )) @@ -265,14 +265,14 @@ def test_LowProfitPairs(mocker, default_conf, fee, caplog): assert not PairLocks.is_global_lock() # Add positive trade - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, min_ago_open=20, min_ago_close=10, profit_rate=1.15, )) assert not freqtrade.protections.stop_per_pair('XRP/BTC') assert not PairLocks.is_pair_locked('XRP/BTC') - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=110, min_ago_close=20, profit_rate=0.8, )) @@ -300,15 +300,15 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not freqtrade.protections.stop_per_pair('XRP/BTC') caplog.clear() - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'NEO/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=1000, min_ago_close=900, profit_rate=1.1, )) @@ -316,7 +316,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not freqtrade.protections.global_stop() assert not freqtrade.protections.stop_per_pair('XRP/BTC') - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=500, min_ago_close=400, profit_rate=0.9, )) @@ -326,7 +326,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not PairLocks.is_pair_locked('XRP/BTC') assert not PairLocks.is_global_lock() - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, min_ago_open=1200, min_ago_close=1100, profit_rate=0.5, )) @@ -339,7 +339,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): assert not log_has_re(message, caplog) # Winning trade ... (should not lock, does not change drawdown!) - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, min_ago_open=320, min_ago_close=410, profit_rate=1.5, )) @@ -349,7 +349,7 @@ def test_MaxDrawdown(mocker, default_conf, fee, caplog): caplog.clear() # Add additional negative trade, causing a loss of > 15% - Trade.session.add(generate_mock_trade( + Trade.query.session.add(generate_mock_trade( 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, min_ago_open=20, min_ago_close=10, profit_rate=0.8, )) diff --git a/tests/rpc/test_fiat_convert.py b/tests/rpc/test_fiat_convert.py index ed21bc516..5174f9416 100644 --- a/tests/rpc/test_fiat_convert.py +++ b/tests/rpc/test_fiat_convert.py @@ -1,44 +1,16 @@ # pragma pylint: disable=missing-docstring, too-many-arguments, too-many-ancestors, # pragma pylint: disable=protected-access, C0103 -import time +import datetime from unittest.mock import MagicMock import pytest from requests.exceptions import RequestException -from freqtrade.rpc.fiat_convert import CryptoFiat, CryptoToFiatConverter +from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from tests.conftest import log_has, log_has_re -def test_pair_convertion_object(): - pair_convertion = CryptoFiat( - crypto_symbol='btc', - fiat_symbol='usd', - price=12345.0 - ) - - # Check the cache duration is 6 hours - assert pair_convertion.CACHE_DURATION == 6 * 60 * 60 - - # Check a regular usage - assert pair_convertion.crypto_symbol == 'btc' - assert pair_convertion.fiat_symbol == 'usd' - assert pair_convertion.price == 12345.0 - assert pair_convertion.is_expired() is False - - # Update the expiration time (- 2 hours) and check the behavior - pair_convertion._expiration = time.time() - 2 * 60 * 60 - assert pair_convertion.is_expired() is True - - # Check set price behaviour - time_reference = time.time() + pair_convertion.CACHE_DURATION - pair_convertion.set_price(price=30000.123) - assert pair_convertion.is_expired() is False - assert pair_convertion._expiration >= time_reference - assert pair_convertion.price == 30000.123 - - def test_fiat_convert_is_supported(mocker): fiat_convert = CryptoToFiatConverter() assert fiat_convert._is_supported_fiat(fiat='USD') is True @@ -47,31 +19,15 @@ def test_fiat_convert_is_supported(mocker): assert fiat_convert._is_supported_fiat(fiat='ABC') is False -def test_fiat_convert_add_pair(mocker): - - fiat_convert = CryptoToFiatConverter() - - pair_len = len(fiat_convert._pairs) - assert pair_len == 0 - - fiat_convert._add_pair(crypto_symbol='btc', fiat_symbol='usd', price=12345.0) - pair_len = len(fiat_convert._pairs) - assert pair_len == 1 - assert fiat_convert._pairs[0].crypto_symbol == 'btc' - assert fiat_convert._pairs[0].fiat_symbol == 'usd' - assert fiat_convert._pairs[0].price == 12345.0 - - fiat_convert._add_pair(crypto_symbol='btc', fiat_symbol='Eur', price=13000.2) - pair_len = len(fiat_convert._pairs) - assert pair_len == 2 - assert fiat_convert._pairs[1].crypto_symbol == 'btc' - assert fiat_convert._pairs[1].fiat_symbol == 'eur' - assert fiat_convert._pairs[1].price == 13000.2 - - def test_fiat_convert_find_price(mocker): fiat_convert = CryptoToFiatConverter() + fiat_convert._cryptomap = {} + fiat_convert._backoff = 0 + mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._load_cryptomap', + return_value=None) + assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='EUR') == 0.0 + with pytest.raises(ValueError, match=r'The fiat ABC is not supported.'): fiat_convert._find_price(crypto_symbol='BTC', fiat_symbol='ABC') @@ -95,8 +51,8 @@ def test_fiat_convert_unsupported_crypto(mocker, caplog): def test_fiat_convert_get_price(mocker): - mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', - return_value=28000.0) + find_price = mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', + return_value=28000.0) fiat_convert = CryptoToFiatConverter() @@ -104,26 +60,17 @@ def test_fiat_convert_get_price(mocker): fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='US Dollar') # Check the value return by the method - pair_len = len(fiat_convert._pairs) + pair_len = len(fiat_convert._pair_price) assert pair_len == 0 assert fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='usd') == 28000.0 - assert fiat_convert._pairs[0].crypto_symbol == 'btc' - assert fiat_convert._pairs[0].fiat_symbol == 'usd' - assert fiat_convert._pairs[0].price == 28000.0 - assert fiat_convert._pairs[0]._expiration != 0 - assert len(fiat_convert._pairs) == 1 + assert fiat_convert._pair_price['btc/usd'] == 28000.0 + assert len(fiat_convert._pair_price) == 1 + assert find_price.call_count == 1 # Verify the cached is used - fiat_convert._pairs[0].price = 9867.543 - expiration = fiat_convert._pairs[0]._expiration + fiat_convert._pair_price['btc/usd'] = 9867.543 assert fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='usd') == 9867.543 - assert fiat_convert._pairs[0]._expiration == expiration - - # Verify the cache expiration - expiration = time.time() - 2 * 60 * 60 - fiat_convert._pairs[0]._expiration = expiration - assert fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='usd') == 28000.0 - assert fiat_convert._pairs[0]._expiration is not expiration + assert find_price.call_count == 1 def test_fiat_convert_same_currencies(mocker): @@ -175,6 +122,28 @@ def test_fiat_convert_without_network(mocker): CryptoToFiatConverter._coingekko = cmc_temp +def test_fiat_too_many_requests_response(mocker, caplog): + # Because CryptoToFiatConverter is a Singleton we reset the listings + req_exception = "429 Too Many Requests" + listmock = MagicMock(return_value="{}", side_effect=RequestException(req_exception)) + mocker.patch.multiple( + 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', + get_coins_list=listmock, + ) + # with pytest.raises(RequestEsxception): + fiat_convert = CryptoToFiatConverter() + fiat_convert._cryptomap = {} + fiat_convert._load_cryptomap() + + length_cryptomap = len(fiat_convert._cryptomap) + assert length_cryptomap == 0 + assert fiat_convert._backoff > datetime.datetime.now().timestamp() + assert log_has( + 'Too many requests for Coingecko API, backing off and trying again later.', + caplog + ) + + def test_fiat_invalid_response(mocker, caplog): # Because CryptoToFiatConverter is a Singleton we reset the listings listmock = MagicMock(return_value="{'novalidjson':DEADBEEFf}") diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 60d9950aa..8a41099f4 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -1,18 +1,19 @@ # pragma pylint: disable=missing-docstring, C0103 # pragma pylint: disable=invalid-sequence-index, invalid-name, too-many-arguments -from datetime import datetime +from datetime import datetime, timedelta, timezone from unittest.mock import ANY, MagicMock, PropertyMock import pytest from numpy import isnan from freqtrade.edge import PairInfo +from freqtrade.enums import State from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError from freqtrade.persistence import Trade +from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter -from freqtrade.state import State from tests.conftest import create_mock_trades, get_patched_freqtradebot, patch_get_signal @@ -52,7 +53,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'pair': 'ETH/BTC', 'base_currency': 'BTC', 'open_date': ANY, - 'open_date_hum': ANY, 'open_timestamp': ANY, 'is_open': ANY, 'fee_open': ANY, @@ -72,7 +72,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'timeframe': 5, 'open_order_id': ANY, 'close_date': None, - 'close_date_hum': None, 'close_timestamp': None, 'open_rate': 1.098e-05, 'close_rate': None, @@ -91,6 +90,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'profit_ratio': -0.00408133, 'profit_pct': -0.41, 'profit_abs': -4.09e-06, + 'profit_fiat': ANY, 'stop_loss_abs': 9.882e-06, 'stop_loss_pct': -10.0, 'stop_loss_ratio': -0.1, @@ -106,10 +106,10 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'stoploss_entry_dist': -0.00010475, 'stoploss_entry_dist_ratio': -0.10448878, 'open_order': None, - 'exchange': 'bittrex', + 'exchange': 'binance', } - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) @@ -119,7 +119,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'pair': 'ETH/BTC', 'base_currency': 'BTC', 'open_date': ANY, - 'open_date_hum': ANY, 'open_timestamp': ANY, 'is_open': ANY, 'fee_open': ANY, @@ -139,7 +138,6 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'timeframe': ANY, 'open_order_id': ANY, 'close_date': None, - 'close_date_hum': None, 'close_timestamp': None, 'open_rate': 1.098e-05, 'close_rate': None, @@ -158,6 +156,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'profit_ratio': ANY, 'profit_pct': ANY, 'profit_abs': ANY, + 'profit_fiat': ANY, 'stop_loss_abs': 9.882e-06, 'stop_loss_pct': -10.0, 'stop_loss_ratio': -0.1, @@ -173,7 +172,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'stoploss_entry_dist': -0.00010475, 'stoploss_entry_dist_ratio': -0.10448878, 'open_order': None, - 'exchange': 'bittrex', + 'exchange': 'binance', } @@ -200,28 +199,31 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: freqtradebot.enter_positions() - result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') + result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert "Since" in headers assert "Pair" in headers assert 'instantly' == result[0][2] assert 'ETH/BTC' in result[0][1] assert '-0.41%' == result[0][3] + assert isnan(fiat_profit_sum) # Test with fiatconvert rpc._fiat_converter = CryptoToFiatConverter() - result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') + result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert "Since" in headers assert "Pair" in headers assert 'instantly' == result[0][2] assert 'ETH/BTC' in result[0][1] assert '-0.41% (-0.06)' == result[0][3] + assert '-0.06' == f'{fiat_profit_sum:.2f}' - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) - result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') + result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] assert 'ETH/BTC' in result[0][1] assert 'nan%' == result[0][3] + assert isnan(fiat_profit_sum) def test_rpc_daily_profit(default_conf, update, ticker, fee, @@ -412,26 +414,26 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) - assert prec_satoshi(stats['profit_closed_percent'], 6.2) + assert prec_satoshi(stats['profit_closed_percent_mean'], 6.2) assert prec_satoshi(stats['profit_closed_fiat'], 0.93255) assert prec_satoshi(stats['profit_all_coin'], 5.802e-05) - assert prec_satoshi(stats['profit_all_percent'], 2.89) + assert prec_satoshi(stats['profit_all_percent_mean'], 2.89) assert prec_satoshi(stats['profit_all_fiat'], 0.8703) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] == '0:00:00' + assert stats['avg_duration'] in ('0:00:00', '0:00:01') assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] == '0:00:00' + assert stats['avg_duration'] in ('0:00:00', '0:00:01') assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) assert isnan(stats['profit_all_coin']) @@ -481,10 +483,10 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert prec_satoshi(stats['profit_closed_coin'], 0) - assert prec_satoshi(stats['profit_closed_percent'], 0) + assert prec_satoshi(stats['profit_closed_percent_mean'], 0) assert prec_satoshi(stats['profit_closed_fiat'], 0) assert prec_satoshi(stats['profit_all_coin'], 0) - assert prec_satoshi(stats['profit_all_percent'], 0) + assert prec_satoshi(stats['profit_all_percent_mean'], 0) assert prec_satoshi(stats['profit_all_fiat'], 0) assert stats['trade_count'] == 1 assert stats['first_trade_date'] == 'just now' @@ -570,6 +572,8 @@ def test_rpc_balance_handle(default_conf, mocker, tickers): result = rpc._rpc_balance(default_conf['stake_currency'], default_conf['fiat_display_currency']) assert prec_satoshi(result['total'], 12.309096315) assert prec_satoshi(result['value'], 184636.44472997) + assert tickers.call_count == 1 + assert tickers.call_args_list[0][1]['cached'] is True assert 'USD' == result['symbol'] assert result['currencies'] == [ {'currency': 'BTC', @@ -911,6 +915,24 @@ def test_rpcforcebuy_disabled(mocker, default_conf) -> None: rpc._rpc_forcebuy(pair, None) +@pytest.mark.usefixtures("init_persistence") +def test_rpc_delete_lock(mocker, default_conf): + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + rpc = RPC(freqtradebot) + pair = 'ETH/BTC' + + PairLocks.lock_pair(pair, datetime.now(timezone.utc) + timedelta(minutes=4)) + PairLocks.lock_pair(pair, datetime.now(timezone.utc) + timedelta(minutes=5)) + PairLocks.lock_pair(pair, datetime.now(timezone.utc) + timedelta(minutes=10)) + locks = rpc._rpc_locks() + assert locks['lock_count'] == 3 + locks1 = rpc._rpc_delete_lock(lockid=locks['locks'][0]['id']) + assert locks1['lock_count'] == 2 + + locks2 = rpc._rpc_delete_lock(pair=pair) + assert locks2['lock_count'] == 0 + + def test_rpc_whitelist(mocker, default_conf) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 263950d83..8d48e2286 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -11,18 +11,20 @@ import uvicorn from fastapi import FastAPI from fastapi.exceptions import HTTPException from fastapi.testclient import TestClient +from numpy import isnan from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ +from freqtrade.enums import RunMode, State +from freqtrade.exceptions import ExchangeError from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC from freqtrade.rpc.api_server import ApiServer from freqtrade.rpc.api_server.api_auth import create_token, get_user_from_token from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer -from freqtrade.state import RunMode, State -from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, log_has_re, - patch_get_signal) +from tests.conftest import (create_mock_trades, get_mock_coro, get_patched_freqtradebot, log_has, + log_has_re, patch_get_signal) BASE_URI = "/api/v1" @@ -228,7 +230,7 @@ def test_api__init__(default_conf, mocker): assert apiserver._config == default_conf -def test_api_UvicornServer(default_conf, mocker): +def test_api_UvicornServer(mocker): thread_mock = mocker.patch('freqtrade.rpc.api_server.uvicorn_threaded.threading.Thread') s = UvicornServer(uvicorn.Config(MagicMock(), port=8080, host='127.0.0.1')) assert thread_mock.call_count == 0 @@ -246,6 +248,38 @@ def test_api_UvicornServer(default_conf, mocker): assert s.should_exit is True +def test_api_UvicornServer_run(mocker): + serve_mock = mocker.patch('freqtrade.rpc.api_server.uvicorn_threaded.UvicornServer.serve', + get_mock_coro(None)) + s = UvicornServer(uvicorn.Config(MagicMock(), port=8080, host='127.0.0.1')) + assert serve_mock.call_count == 0 + + s.install_signal_handlers() + # Original implementation starts a thread - make sure that's not the case + assert serve_mock.call_count == 0 + + # Fake started to avoid sleeping forever + s.started = True + s.run() + assert serve_mock.call_count == 1 + + +def test_api_UvicornServer_run_no_uvloop(mocker, import_fails): + serve_mock = mocker.patch('freqtrade.rpc.api_server.uvicorn_threaded.UvicornServer.serve', + get_mock_coro(None)) + s = UvicornServer(uvicorn.Config(MagicMock(), port=8080, host='127.0.0.1')) + assert serve_mock.call_count == 0 + + s.install_signal_handlers() + # Original implementation starts a thread - make sure that's not the case + assert serve_mock.call_count == 0 + + # Fake started to avoid sleeping forever + s.started = True + s.run() + assert serve_mock.call_count == 1 + + def test_api_run(default_conf, mocker, caplog): default_conf.update({"api_server": {"enabled": True, "listen_ip_address": "127.0.0.1", @@ -295,7 +329,7 @@ def test_api_run(default_conf, mocker, caplog): "Please make sure that this is intentional!", caplog) assert log_has_re("SECURITY WARNING - `jwt_secret_key` seems to be default.*", caplog) - # Test crashing flask + # Test crashing API server caplog.clear() mocker.patch('freqtrade.rpc.api_server.webserver.UvicornServer', MagicMock(side_effect=Exception)) @@ -382,10 +416,10 @@ def test_api_count(botclient, mocker, ticker, fee, markets): assert rc.json()["max"] == 1 # Create some test data - ftbot.enter_positions() + create_mock_trades(fee) rc = client_get(client, f"{BASE_URI}/count") assert_response(rc) - assert rc.json()["current"] == 1 + assert rc.json()["current"] == 4 assert rc.json()["max"] == 1 ftbot.config['max_open_trades'] = float('inf') @@ -416,6 +450,16 @@ def test_api_locks(botclient): assert 'randreason' in (rc.json()['locks'][0]['reason'], rc.json()['locks'][1]['reason']) assert 'deadbeef' in (rc.json()['locks'][0]['reason'], rc.json()['locks'][1]['reason']) + # Test deletions + rc = client_delete(client, f"{BASE_URI}/locks/1") + assert_response(rc) + assert rc.json()['lock_count'] == 1 + + rc = client_post(client, f"{BASE_URI}/locks/delete", + data='{"pair": "XRP/BTC"}') + assert_response(rc) + assert rc.json()['lock_count'] == 0 + def test_api_show_config(botclient, mocker): ftbot, client = botclient @@ -424,7 +468,7 @@ def test_api_show_config(botclient, mocker): rc = client_get(client, f"{BASE_URI}/show_config") assert_response(rc) assert 'dry_run' in rc.json() - assert rc.json()['exchange'] == 'bittrex' + assert rc.json()['exchange'] == 'binance' assert rc.json()['timeframe'] == '5m' assert rc.json()['timeframe_ms'] == 300000 assert rc.json()['timeframe_min'] == 5 @@ -462,20 +506,43 @@ def test_api_trades(botclient, mocker, fee, markets): ) rc = client_get(client, f"{BASE_URI}/trades") assert_response(rc) - assert len(rc.json()) == 2 + assert len(rc.json()) == 3 assert rc.json()['trades_count'] == 0 + assert rc.json()['total_trades'] == 0 create_mock_trades(fee) - Trade.session.flush() + Trade.query.session.flush() rc = client_get(client, f"{BASE_URI}/trades") assert_response(rc) assert len(rc.json()['trades']) == 2 assert rc.json()['trades_count'] == 2 + assert rc.json()['total_trades'] == 2 rc = client_get(client, f"{BASE_URI}/trades?limit=1") assert_response(rc) assert len(rc.json()['trades']) == 1 assert rc.json()['trades_count'] == 1 + assert rc.json()['total_trades'] == 2 + + +def test_api_trade_single(botclient, mocker, fee, ticker, markets): + ftbot, client = botclient + patch_get_signal(ftbot, (True, False)) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + fetch_ticker=ticker, + ) + rc = client_get(client, f"{BASE_URI}/trade/3") + assert_response(rc, 404) + assert rc.json()['detail'] == 'Trade not found.' + + create_mock_trades(fee) + Trade.query.session.flush() + + rc = client_get(client, f"{BASE_URI}/trade/3") + assert_response(rc) + assert rc.json()['trade_id'] == 3 def test_api_delete_trade(botclient, mocker, fee, markets): @@ -494,7 +561,7 @@ def test_api_delete_trade(botclient, mocker, fee, markets): assert_response(rc, 502) create_mock_trades(fee) - Trade.session.flush() + Trade.query.session.flush() ftbot.strategy.order_types['stoploss_on_exchange'] = True trades = Trade.query.all() trades[1].stoploss_order_id = '1234' @@ -568,7 +635,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): @pytest.mark.usefixtures("init_persistence") -def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, limit_sell_order): +def test_api_profit(botclient, mocker, ticker, fee, markets): ftbot, client = botclient patch_get_signal(ftbot, (True, False)) mocker.patch.multiple( @@ -583,50 +650,33 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li assert_response(rc, 200) assert rc.json()['trade_count'] == 0 - ftbot.enter_positions() - trade = Trade.query.first() - + create_mock_trades(fee) # Simulate fulfilled LIMIT_BUY order for trade - trade.update(limit_buy_order) - rc = client_get(client, f"{BASE_URI}/profit") - assert_response(rc, 200) - # One open trade - assert rc.json()['trade_count'] == 1 - assert rc.json()['best_pair'] == '' - assert rc.json()['best_rate'] == 0 - - trade = Trade.query.first() - trade.update(limit_sell_order) - - trade.close_date = datetime.utcnow() - trade.is_open = False rc = client_get(client, f"{BASE_URI}/profit") assert_response(rc) assert rc.json() == {'avg_duration': ANY, - 'best_pair': 'ETH/BTC', - 'best_rate': 6.2, - 'first_trade_date': 'just now', + 'best_pair': 'XRP/BTC', + 'best_rate': 1.0, + 'first_trade_date': ANY, 'first_trade_timestamp': ANY, - 'latest_trade_date': 'just now', + 'latest_trade_date': '5 minutes ago', 'latest_trade_timestamp': ANY, - 'profit_all_coin': 6.217e-05, - 'profit_all_fiat': 0.76748865, - 'profit_all_percent': 6.2, - 'profit_all_percent_mean': 6.2, - 'profit_all_ratio_mean': 0.06201058, - 'profit_all_percent_sum': 6.2, - 'profit_all_ratio_sum': 0.06201058, - 'profit_closed_coin': 6.217e-05, - 'profit_closed_fiat': 0.76748865, - 'profit_closed_percent': 6.2, - 'profit_closed_ratio_mean': 0.06201058, - 'profit_closed_percent_mean': 6.2, - 'profit_closed_ratio_sum': 0.06201058, - 'profit_closed_percent_sum': 6.2, - 'trade_count': 1, - 'closed_trade_count': 1, - 'winning_trades': 1, + 'profit_all_coin': -44.0631579, + 'profit_all_fiat': -543959.6842755, + 'profit_all_percent_mean': -66.41, + 'profit_all_ratio_mean': -0.6641100666666667, + 'profit_all_percent_sum': -398.47, + 'profit_all_ratio_sum': -3.9846604, + 'profit_closed_coin': 0.00073913, + 'profit_closed_fiat': 9.124559849999999, + 'profit_closed_ratio_mean': 0.0075, + 'profit_closed_percent_mean': 0.75, + 'profit_closed_ratio_sum': 0.015, + 'profit_closed_percent_sum': 1.5, + 'trade_count': 6, + 'closed_trade_count': 2, + 'winning_trades': 2, 'losing_trades': 0, } @@ -660,7 +710,7 @@ def test_api_stats(botclient, mocker, ticker, fee, markets,): assert 'draws' in rc.json()['durations'] -def test_api_performance(botclient, mocker, ticker, fee): +def test_api_performance(botclient, fee): ftbot, client = botclient patch_get_signal(ftbot, (True, False)) @@ -678,7 +728,8 @@ def test_api_performance(botclient, mocker, ticker, fee): ) trade.close_profit = trade.calc_profit_ratio() - Trade.session.add(trade) + trade.close_profit_abs = trade.calc_profit() + Trade.query.session.add(trade) trade = Trade( pair='XRP/ETH', @@ -693,14 +744,16 @@ def test_api_performance(botclient, mocker, ticker, fee): close_rate=0.391 ) trade.close_profit = trade.calc_profit_ratio() - Trade.session.add(trade) - Trade.session.flush() + trade.close_profit_abs = trade.calc_profit() + + Trade.query.session.add(trade) + Trade.query.session.flush() rc = client_get(client, f"{BASE_URI}/performance") assert_response(rc) assert len(rc.json()) == 2 - assert rc.json() == [{'count': 1, 'pair': 'LTC/ETH', 'profit': 7.61}, - {'count': 1, 'pair': 'XRP/ETH', 'profit': -5.57}] + assert rc.json() == [{'count': 1, 'pair': 'LTC/ETH', 'profit': 7.61, 'profit_abs': 0.01872279}, + {'count': 1, 'pair': 'XRP/ETH', 'profit': -5.57, 'profit_abs': -0.1150375}] def test_api_status(botclient, mocker, ticker, fee, markets): @@ -711,83 +764,84 @@ def test_api_status(botclient, mocker, ticker, fee, markets): get_balances=MagicMock(return_value=ticker), fetch_ticker=ticker, get_fee=fee, - markets=PropertyMock(return_value=markets) + markets=PropertyMock(return_value=markets), + fetch_order=MagicMock(return_value={}), ) rc = client_get(client, f"{BASE_URI}/status") assert_response(rc, 200) assert rc.json() == [] - - ftbot.enter_positions() - trades = Trade.get_open_trades() - trades[0].open_order_id = None - ftbot.exit_positions(trades) - Trade.session.flush() + create_mock_trades(fee) rc = client_get(client, f"{BASE_URI}/status") assert_response(rc) - assert len(rc.json()) == 1 - assert rc.json() == [{ - 'amount': 91.07468123, - 'amount_requested': 91.07468123, - 'base_currency': 'BTC', + assert len(rc.json()) == 4 + assert rc.json()[0] == { + 'amount': 123.0, + 'amount_requested': 123.0, 'close_date': None, - 'close_date_hum': None, 'close_timestamp': None, 'close_profit': None, 'close_profit_pct': None, 'close_profit_abs': None, 'close_rate': None, - 'current_profit': -0.00408133, - 'current_profit_pct': -0.41, - 'current_profit_abs': -4.09e-06, - 'profit_ratio': -0.00408133, - 'profit_pct': -0.41, - 'profit_abs': -4.09e-06, + 'current_profit': ANY, + 'current_profit_pct': ANY, + 'current_profit_abs': ANY, + 'profit_ratio': ANY, + 'profit_pct': ANY, + 'profit_abs': ANY, + 'profit_fiat': ANY, 'current_rate': 1.099e-05, 'open_date': ANY, - 'open_date_hum': 'just now', 'open_timestamp': ANY, 'open_order': None, - 'open_rate': 1.098e-05, + 'open_rate': 0.123, 'pair': 'ETH/BTC', 'stake_amount': 0.001, - 'stop_loss_abs': 9.882e-06, - 'stop_loss_pct': -10.0, - 'stop_loss_ratio': -0.1, + 'stop_loss_abs': ANY, + 'stop_loss_pct': ANY, + 'stop_loss_ratio': ANY, 'stoploss_order_id': None, 'stoploss_last_update': ANY, 'stoploss_last_update_timestamp': ANY, - 'initial_stop_loss_abs': 9.882e-06, - 'initial_stop_loss_pct': -10.0, - 'initial_stop_loss_ratio': -0.1, - 'stoploss_current_dist': -1.1080000000000002e-06, - 'stoploss_current_dist_ratio': -0.10081893, - 'stoploss_current_dist_pct': -10.08, - 'stoploss_entry_dist': -0.00010475, - 'stoploss_entry_dist_ratio': -0.10448878, + 'initial_stop_loss_abs': 0.0, + 'initial_stop_loss_pct': ANY, + 'initial_stop_loss_ratio': ANY, + 'stoploss_current_dist': ANY, + 'stoploss_current_dist_ratio': ANY, + 'stoploss_current_dist_pct': ANY, + 'stoploss_entry_dist': ANY, + 'stoploss_entry_dist_ratio': ANY, 'trade_id': 1, - 'close_rate_requested': None, - 'current_rate': 1.099e-05, + 'close_rate_requested': ANY, 'fee_close': 0.0025, 'fee_close_cost': None, 'fee_close_currency': None, 'fee_open': 0.0025, 'fee_open_cost': None, 'fee_open_currency': None, - 'open_date': ANY, 'is_open': True, - 'max_rate': 1.099e-05, - 'min_rate': 1.098e-05, - 'open_order_id': None, - 'open_rate_requested': 1.098e-05, - 'open_trade_value': 0.0010025, + 'max_rate': ANY, + 'min_rate': ANY, + 'open_order_id': 'dry_run_buy_12345', + 'open_rate_requested': ANY, + 'open_trade_value': 15.1668225, 'sell_reason': None, 'sell_order_status': None, 'strategy': 'DefaultStrategy', 'timeframe': 5, - 'exchange': 'bittrex', - }] + 'exchange': 'binance', + } + + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', + MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) + + rc = client_get(client, f"{BASE_URI}/status") + assert_response(rc) + resp_values = rc.json() + assert len(resp_values) == 4 + assert isnan(resp_values[0]['profit_abs']) def test_api_version(botclient): @@ -868,7 +922,7 @@ def test_api_forcebuy(botclient, mocker, fee): pair='ETH/ETH', amount=1, amount_requested=1, - exchange='bittrex', + exchange='binance', stake_amount=1, open_rate=0.245441, open_order_id="123456", @@ -891,11 +945,9 @@ def test_api_forcebuy(botclient, mocker, fee): 'amount_requested': 1, 'trade_id': 22, 'close_date': None, - 'close_date_hum': None, 'close_timestamp': None, 'close_rate': 0.265441, 'open_date': ANY, - 'open_date_hum': 'just now', 'open_timestamp': ANY, 'open_rate': 0.245441, 'pair': 'ETH/ETH', @@ -916,6 +968,7 @@ def test_api_forcebuy(botclient, mocker, fee): 'profit_ratio': None, 'profit_pct': None, 'profit_abs': None, + 'profit_fiat': None, 'fee_close': 0.0025, 'fee_close_cost': None, 'fee_close_currency': None, @@ -932,7 +985,7 @@ def test_api_forcebuy(botclient, mocker, fee): 'sell_order_status': None, 'strategy': 'DefaultStrategy', 'timeframe': 5, - 'exchange': 'bittrex', + 'exchange': 'binance', } @@ -1092,6 +1145,14 @@ def test_api_plot_config(botclient): assert_response(rc) assert rc.json() == ftbot.strategy.plot_config assert isinstance(rc.json()['main_plot'], dict) + assert isinstance(rc.json()['subplots'], dict) + + ftbot.strategy.plot_config = {'main_plot': {'sma': {}}} + rc = client_get(client, f"{BASE_URI}/plot_config") + assert_response(rc) + + assert isinstance(rc.json()['main_plot'], dict) + assert isinstance(rc.json()['subplots'], dict) def test_api_strategies(botclient): @@ -1100,7 +1161,11 @@ def test_api_strategies(botclient): rc = client_get(client, f"{BASE_URI}/strategies") assert_response(rc) - assert rc.json() == {'strategies': ['DefaultStrategy', 'TestStrategyLegacy']} + assert rc.json() == {'strategies': [ + 'DefaultStrategy', + 'HyperoptableStrategy', + 'TestStrategyLegacy' + ]} def test_api_strategy(botclient): diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index 3068e9764..918022386 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -3,7 +3,8 @@ import logging import time from unittest.mock import MagicMock -from freqtrade.rpc import RPCManager, RPCMessageType +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPCManager from tests.conftest import get_patched_freqtradebot, log_has @@ -71,7 +72,7 @@ def test_send_msg_telegram_disabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) rpc_manager.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, + 'type': RPCMessageType.STATUS, 'status': 'test' }) @@ -86,7 +87,7 @@ def test_send_msg_telegram_enabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) rpc_manager.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, + 'type': RPCMessageType.STATUS, 'status': 'test' }) @@ -124,7 +125,7 @@ def test_send_msg_webhook_CustomMessagetype(mocker, default_conf, caplog) -> Non rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] - rpc_manager.send_msg({'type': RPCMessageType.STARTUP_NOTIFICATION, + rpc_manager.send_msg({'type': RPCMessageType.STARTUP, 'status': 'TestMessage'}) assert log_has( "Message type 'startup' not implemented by handler webhook.", @@ -140,7 +141,7 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: rpc_manager.startup_messages(default_conf, freqtradebot.pairlists, freqtradebot.protections) assert telegram_mock.call_count == 3 - assert "*Exchange:* `bittrex`" in telegram_mock.call_args_list[1][0][0]['status'] + assert "*Exchange:* `binance`" in telegram_mock.call_args_list[1][0][0]['status'] telegram_mock.reset_mock() default_conf['dry_run'] = True diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index f065bb4c5..c4f901208 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -4,6 +4,7 @@ import re from datetime import datetime +from functools import reduce from random import choice, randint from string import ascii_uppercase from unittest.mock import ANY, MagicMock @@ -16,14 +17,13 @@ from telegram.error import NetworkError from freqtrade import __version__ from freqtrade.constants import CANCEL_REASON from freqtrade.edge import PairInfo +from freqtrade.enums import RPCMessageType, RunMode, SellType, State from freqtrade.exceptions import OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.loggers import setup_logging from freqtrade.persistence import PairLocks, Trade -from freqtrade.rpc import RPC, RPCMessageType +from freqtrade.rpc import RPC from freqtrade.rpc.telegram import Telegram, authorized_only -from freqtrade.state import RunMode, State -from freqtrade.strategy.interface import SellType from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, patch_exchange, patch_get_signal, patch_whitelist) @@ -92,7 +92,8 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: message_str = ("rpc.telegram is listening for following commands: [['status'], ['profit'], " "['balance'], ['start'], ['stop'], ['forcesell'], ['forcebuy'], ['trades'], " "['delete'], ['performance'], ['stats'], ['daily'], ['count'], ['locks'], " - "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], ['stopbuy'], " + "['unlock', 'delete_locks'], ['reload_config', 'reload_conf'], " + "['show_config', 'show_conf'], ['stopbuy'], " "['whitelist'], ['blacklist'], ['logs'], ['edge'], ['help'], ['version']" "]") @@ -176,9 +177,7 @@ def test_telegram_status(default_conf, update, mocker) -> None: 'pair': 'ETH/BTC', 'base_currency': 'BTC', 'open_date': arrow.utcnow(), - 'open_date_hum': arrow.utcnow().humanize, 'close_date': None, - 'close_date_hum': None, 'open_rate': 1.099e-05, 'close_rate': None, 'current_rate': 1.098e-05, @@ -443,8 +442,10 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) - - telegram._profit(update=update, context=MagicMock()) + context = MagicMock() + # Test with invalid 2nd argument (should silently pass) + context.args = ["aaa"] + telegram._profit(update=update, context=context) assert msg_mock.call_count == 1 assert 'No closed trade' in msg_mock.call_args_list[-1][0][0] assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] @@ -520,7 +521,7 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick assert 'Balance:' in result assert 'Est. BTC:' in result assert 'BTC: 12.00000000' in result - assert '*XRP:* not showing <1$ amount' in result + assert '*XRP:* not showing <0.0001 BTC amount' in result def test_balance_handle_empty_response(default_conf, update, mocker) -> None: @@ -684,12 +685,12 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, context.args = ["1"] telegram._forcesell(update=update, context=context) - assert msg_mock.call_count == 3 + assert msg_mock.call_count == 4 last_msg = msg_mock.call_args_list[-1][0][0] assert { - 'type': RPCMessageType.SELL_NOTIFICATION, + 'type': RPCMessageType.SELL, 'trade_id': 1, - 'exchange': 'Bittrex', + 'exchange': 'Binance', 'pair': 'ETH/BTC', 'gain': 'profit', 'limit': 1.173e-05, @@ -704,6 +705,7 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, 'sell_reason': SellType.FORCE_SELL.value, 'open_date': ANY, 'close_date': ANY, + 'close_rate': ANY, } == last_msg @@ -744,13 +746,13 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, context.args = ["1"] telegram._forcesell(update=update, context=context) - assert msg_mock.call_count == 3 + assert msg_mock.call_count == 4 last_msg = msg_mock.call_args_list[-1][0][0] assert { - 'type': RPCMessageType.SELL_NOTIFICATION, + 'type': RPCMessageType.SELL, 'trade_id': 1, - 'exchange': 'Bittrex', + 'exchange': 'Binance', 'pair': 'ETH/BTC', 'gain': 'loss', 'limit': 1.043e-05, @@ -765,6 +767,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, 'sell_reason': SellType.FORCE_SELL.value, 'open_date': ANY, 'close_date': ANY, + 'close_rate': ANY, } == last_msg @@ -795,13 +798,13 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None context.args = ["all"] telegram._forcesell(update=update, context=context) - # Called for each trade 3 times - assert msg_mock.call_count == 8 - msg = msg_mock.call_args_list[1][0][0] + # Called for each trade 4 times + assert msg_mock.call_count == 12 + msg = msg_mock.call_args_list[2][0][0] assert { - 'type': RPCMessageType.SELL_NOTIFICATION, + 'type': RPCMessageType.SELL, 'trade_id': 1, - 'exchange': 'Bittrex', + 'exchange': 'Binance', 'pair': 'ETH/BTC', 'gain': 'loss', 'limit': 1.099e-05, @@ -816,6 +819,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'sell_reason': SellType.FORCE_SELL.value, 'open_date': ANY, 'close_date': ANY, + 'close_rate': ANY, } == msg @@ -900,6 +904,33 @@ def test_forcebuy_handle_exception(default_conf, update, mocker) -> None: assert msg_mock.call_args_list[0][0][0] == 'Forcebuy not enabled.' +def test_forcebuy_no_pair(default_conf, update, mocker) -> None: + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) + + fbuy_mock = MagicMock(return_value=None) + mocker.patch('freqtrade.rpc.RPC._rpc_forcebuy', fbuy_mock) + + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + + patch_get_signal(freqtradebot, (True, False)) + + context = MagicMock() + context.args = [] + telegram._forcebuy(update=update, context=context) + + assert fbuy_mock.call_count == 0 + assert msg_mock.call_count == 1 + assert msg_mock.call_args_list[0][1]['msg'] == 'Which pair?' + # assert msg_mock.call_args_list[0][1]['callback_query_handler'] == 'forcebuy' + keyboard = msg_mock.call_args_list[0][1]['keyboard'] + assert reduce(lambda acc, x: acc + len(x), keyboard, 0) == 4 + update = MagicMock() + update.callback_query = MagicMock() + update.callback_query.data = 'XRP/USDT' + telegram._forcebuy_inline(update, None) + assert fbuy_mock.call_count == 1 + + def test_performance_handle(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, mocker) -> None: @@ -927,7 +958,7 @@ def test_performance_handle(default_conf, update, ticker, fee, telegram._performance(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert 'Performance' in msg_mock.call_args_list[0][0][0] - assert 'ETH/BTC\t6.20% (1)' in msg_mock.call_args_list[0][0][0] + assert 'ETH/BTC\t0.00006217 BTC (6.20%) (1)' in msg_mock.call_args_list[0][0][0] def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: @@ -967,6 +998,11 @@ def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) -> None ) telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) + telegram._locks(update=update, context=MagicMock()) + assert msg_mock.call_count == 1 + assert 'No active locks.' in msg_mock.call_args_list[0][0][0] + + msg_mock.reset_mock() PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason') PairLocks.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef') @@ -981,6 +1017,16 @@ def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) -> None assert 'deadbeef' in msg_mock.call_args_list[0][0][0] assert 'randreason' in msg_mock.call_args_list[0][0][0] + context = MagicMock() + context.args = ['XRP/BTC'] + msg_mock.reset_mock() + telegram._delete_locks(update=update, context=context) + + assert 'ETH/BTC' in msg_mock.call_args_list[0][0][0] + assert 'randreason' in msg_mock.call_args_list[0][0][0] + assert 'XRP/BTC' not in msg_mock.call_args_list[0][0][0] + assert 'deadbeef' not in msg_mock.call_args_list[0][0][0] + def test_whitelist_static(default_conf, update, mocker) -> None: @@ -1090,6 +1136,15 @@ def test_edge_enabled(edge_conf, update, mocker) -> None: assert 'Edge only validated following pairs:\n
' in msg_mock.call_args_list[0][0][0]
     assert 'Pair      Winrate    Expectancy    Stoploss' in msg_mock.call_args_list[0][0][0]
 
+    msg_mock.reset_mock()
+
+    mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock(
+        return_value={}))
+    telegram._edge(update=update, context=MagicMock())
+    assert msg_mock.call_count == 1
+    assert 'Edge only validated following pairs:' in msg_mock.call_args_list[0][0][0]
+    assert 'Winrate' not in msg_mock.call_args_list[0][0][0]
+
 
 def test_telegram_trades(mocker, update, default_conf, fee):
 
@@ -1117,8 +1172,10 @@ def test_telegram_trades(mocker, update, default_conf, fee):
     msg_mock.call_count == 1
     assert "2 recent trades:" in msg_mock.call_args_list[0][0][0]
     assert "Profit (" in msg_mock.call_args_list[0][0][0]
-    assert "Open Date" in msg_mock.call_args_list[0][0][0]
+    assert "Close Date" in msg_mock.call_args_list[0][0][0]
     assert "
" in msg_mock.call_args_list[0][0][0]
+    assert bool(re.search(r"just now[ ]*XRP\/BTC \(#3\)  1.00% \(",
+                msg_mock.call_args_list[0][0][0]))
 
 
 def test_telegram_delete_trade(mocker, update, default_conf, fee):
@@ -1167,7 +1224,7 @@ def test_show_config_handle(default_conf, update, mocker) -> None:
     telegram._show_config(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
     assert '*Mode:* `{}`'.format('Dry-run') in msg_mock.call_args_list[0][0][0]
-    assert '*Exchange:* `bittrex`' in msg_mock.call_args_list[0][0][0]
+    assert '*Exchange:* `binance`' in msg_mock.call_args_list[0][0][0]
     assert '*Strategy:* `DefaultStrategy`' in msg_mock.call_args_list[0][0][0]
     assert '*Stoploss:* `-0.1`' in msg_mock.call_args_list[0][0][0]
 
@@ -1176,7 +1233,7 @@ def test_show_config_handle(default_conf, update, mocker) -> None:
     telegram._show_config(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
     assert '*Mode:* `{}`'.format('Dry-run') in msg_mock.call_args_list[0][0][0]
-    assert '*Exchange:* `bittrex`' in msg_mock.call_args_list[0][0][0]
+    assert '*Exchange:* `binance`' in msg_mock.call_args_list[0][0][0]
     assert '*Strategy:* `DefaultStrategy`' in msg_mock.call_args_list[0][0][0]
     assert '*Initial Stoploss:* `-0.1`' in msg_mock.call_args_list[0][0][0]
 
@@ -1184,8 +1241,9 @@ def test_show_config_handle(default_conf, update, mocker) -> None:
 def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None:
 
     msg = {
-        'type': RPCMessageType.BUY_NOTIFICATION,
-        'exchange': 'Bittrex',
+        'type': RPCMessageType.BUY,
+        'trade_id': 1,
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'limit': 1.099e-05,
         'order_type': 'limit',
@@ -1201,7 +1259,7 @@ def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None:
 
     telegram.send_msg(msg)
     assert msg_mock.call_args[0][0] \
-        == '\N{LARGE BLUE CIRCLE} *Bittrex:* Buying ETH/BTC\n' \
+        == '\N{LARGE BLUE CIRCLE} *Binance:* Buying ETH/BTC (#1)\n' \
            '*Amount:* `1333.33333333`\n' \
            '*Open Rate:* `0.00001099`\n' \
            '*Current Rate:* `0.00001099`\n' \
@@ -1228,13 +1286,34 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None:
     telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram.send_msg({
-        'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
-        'exchange': 'Bittrex',
+        'type': RPCMessageType.BUY_CANCEL,
+        'trade_id': 1,
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'reason': CANCEL_REASON['TIMEOUT']
     })
-    assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Bittrex:* '
-            'Cancelling open buy Order for ETH/BTC. Reason: cancelled due to timeout.')
+    assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Binance:* '
+            'Cancelling open buy Order for ETH/BTC (#1). '
+            'Reason: cancelled due to timeout.')
+
+
+def test_send_msg_buy_fill_notification(default_conf, mocker) -> None:
+
+    default_conf['telegram']['notification_settings']['buy_fill'] = 'on'
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
+    telegram.send_msg({
+        'type': RPCMessageType.BUY_FILL,
+        'trade_id': 1,
+        'exchange': 'Binance',
+        'pair': 'ETH/USDT',
+        'open_rate': 200,
+        'stake_amount': 100,
+        'amount': 0.5,
+        'open_date': arrow.utcnow().datetime
+    })
+    assert (msg_mock.call_args[0][0] == '\N{LARGE CIRCLE} *Binance:* '
+            'Buy order for ETH/USDT (#1) filled for 200.')
 
 
 def test_send_msg_sell_notification(default_conf, mocker) -> None:
@@ -1244,7 +1323,8 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None:
     old_convamount = telegram._rpc._fiat_converter.convert_amount
     telegram._rpc._fiat_converter.convert_amount = lambda a, b, c: -24.812
     telegram.send_msg({
-        'type': RPCMessageType.SELL_NOTIFICATION,
+        'type': RPCMessageType.SELL,
+        'trade_id': 1,
         'exchange': 'Binance',
         'pair': 'KEY/ETH',
         'gain': 'loss',
@@ -1262,18 +1342,20 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None:
         'close_date': arrow.utcnow(),
     })
     assert msg_mock.call_args[0][0] \
-        == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH\n'
+        == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n'
+            '*Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n'
+            '*Sell Reason:* `stop_loss`\n'
+            '*Duration:* `1:00:00 (60.0 min)`\n'
             '*Amount:* `1333.33333333`\n'
             '*Open Rate:* `0.00007500`\n'
             '*Current Rate:* `0.00003201`\n'
-            '*Close Rate:* `0.00003201`\n'
-            '*Sell Reason:* `stop_loss`\n'
-            '*Duration:* `1:00:00 (60.0 min)`\n'
-            '*Profit:* `-57.41%` `(loss: -0.05746268 ETH / -24.812 USD)`')
+            '*Close Rate:* `0.00003201`'
+            )
 
     msg_mock.reset_mock()
     telegram.send_msg({
-        'type': RPCMessageType.SELL_NOTIFICATION,
+        'type': RPCMessageType.SELL,
+        'trade_id': 1,
         'exchange': 'Binance',
         'pair': 'KEY/ETH',
         'gain': 'loss',
@@ -1290,14 +1372,15 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None:
         'close_date': arrow.utcnow(),
     })
     assert msg_mock.call_args[0][0] \
-        == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH\n'
+        == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n'
+            '*Profit:* `-57.41%`\n'
+            '*Sell Reason:* `stop_loss`\n'
+            '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n'
             '*Amount:* `1333.33333333`\n'
             '*Open Rate:* `0.00007500`\n'
             '*Current Rate:* `0.00003201`\n'
-            '*Close Rate:* `0.00003201`\n'
-            '*Sell Reason:* `stop_loss`\n'
-            '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n'
-            '*Profit:* `-57.41%`')
+            '*Close Rate:* `0.00003201`'
+            )
     # Reset singleton function to avoid random breaks
     telegram._rpc._fiat_converter.convert_amount = old_convamount
 
@@ -1309,33 +1392,65 @@ def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None:
     old_convamount = telegram._rpc._fiat_converter.convert_amount
     telegram._rpc._fiat_converter.convert_amount = lambda a, b, c: -24.812
     telegram.send_msg({
-        'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
+        'type': RPCMessageType.SELL_CANCEL,
+        'trade_id': 1,
         'exchange': 'Binance',
         'pair': 'KEY/ETH',
         'reason': 'Cancelled on exchange'
     })
     assert msg_mock.call_args[0][0] \
-        == ('\N{WARNING SIGN} *Binance:* Cancelling Open Sell Order for KEY/ETH. '
-            'Reason: Cancelled on exchange')
+        == ('\N{WARNING SIGN} *Binance:* Cancelling open sell Order for KEY/ETH (#1).'
+            ' Reason: Cancelled on exchange.')
 
     msg_mock.reset_mock()
     telegram.send_msg({
-        'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
+        'type': RPCMessageType.SELL_CANCEL,
+        'trade_id': 1,
         'exchange': 'Binance',
         'pair': 'KEY/ETH',
         'reason': 'timeout'
     })
     assert msg_mock.call_args[0][0] \
-        == ('\N{WARNING SIGN} *Binance:* Cancelling Open Sell Order for KEY/ETH. Reason: timeout')
+        == ('\N{WARNING SIGN} *Binance:* Cancelling open sell Order for KEY/ETH (#1).'
+            ' Reason: timeout.')
     # Reset singleton function to avoid random breaks
     telegram._rpc._fiat_converter.convert_amount = old_convamount
 
 
+def test_send_msg_sell_fill_notification(default_conf, mocker) -> None:
+
+    default_conf['telegram']['notification_settings']['sell_fill'] = 'on'
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
+    telegram.send_msg({
+        'type': RPCMessageType.SELL_FILL,
+        'trade_id': 1,
+        'exchange': 'Binance',
+        'pair': 'ETH/USDT',
+        'gain': 'loss',
+        'limit': 3.201e-05,
+        'amount': 0.1,
+        'order_type': 'market',
+        'open_rate': 500,
+        'close_rate': 550,
+        'current_rate': 3.201e-05,
+        'profit_amount': -0.05746268,
+        'profit_ratio': -0.57405275,
+        'stake_currency': 'ETH',
+        'fiat_currency': 'USD',
+        'sell_reason': SellType.STOP_LOSS.value,
+        'open_date': arrow.utcnow().shift(hours=-1),
+        'close_date': arrow.utcnow(),
+    })
+    assert msg_mock.call_args[0][0] \
+        == ('\N{LARGE CIRCLE} *Binance:* Sell order for ETH/USDT (#1) filled for 550.')
+
+
 def test_send_msg_status_notification(default_conf, mocker) -> None:
 
     telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     telegram.send_msg({
-        'type': RPCMessageType.STATUS_NOTIFICATION,
+        'type': RPCMessageType.STATUS,
         'status': 'running'
     })
     assert msg_mock.call_args[0][0] == '*Status:* `running`'
@@ -1344,7 +1459,7 @@ def test_send_msg_status_notification(default_conf, mocker) -> None:
 def test_warning_notification(default_conf, mocker) -> None:
     telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     telegram.send_msg({
-        'type': RPCMessageType.WARNING_NOTIFICATION,
+        'type': RPCMessageType.WARNING,
         'status': 'message'
     })
     assert msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Warning:* `message`'
@@ -1353,7 +1468,7 @@ def test_warning_notification(default_conf, mocker) -> None:
 def test_startup_notification(default_conf, mocker) -> None:
     telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     telegram.send_msg({
-        'type': RPCMessageType.STARTUP_NOTIFICATION,
+        'type': RPCMessageType.STARTUP,
         'status': '*Custom:* `Hello World`'
     })
     assert msg_mock.call_args[0][0] == '*Custom:* `Hello World`'
@@ -1372,8 +1487,9 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None:
     telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram.send_msg({
-        'type': RPCMessageType.BUY_NOTIFICATION,
-        'exchange': 'Bittrex',
+        'type': RPCMessageType.BUY,
+        'trade_id': 1,
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'limit': 1.099e-05,
         'order_type': 'limit',
@@ -1385,7 +1501,7 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None:
         'amount': 1333.3333333333335,
         'open_date': arrow.utcnow().shift(hours=-1)
     })
-    assert msg_mock.call_args[0][0] == ('\N{LARGE BLUE CIRCLE} *Bittrex:* Buying ETH/BTC\n'
+    assert msg_mock.call_args[0][0] == ('\N{LARGE BLUE CIRCLE} *Binance:* Buying ETH/BTC (#1)\n'
                                         '*Amount:* `1333.33333333`\n'
                                         '*Open Rate:* `0.00001099`\n'
                                         '*Current Rate:* `0.00001099`\n'
@@ -1397,7 +1513,8 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None:
     telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram.send_msg({
-        'type': RPCMessageType.SELL_NOTIFICATION,
+        'type': RPCMessageType.SELL,
+        'trade_id': 1,
         'exchange': 'Binance',
         'pair': 'KEY/ETH',
         'gain': 'loss',
@@ -1414,14 +1531,15 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None:
         'open_date': arrow.utcnow().shift(hours=-2, minutes=-35, seconds=-3),
         'close_date': arrow.utcnow(),
     })
-    assert msg_mock.call_args[0][0] == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH\n'
+    assert msg_mock.call_args[0][0] == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n'
+                                        '*Profit:* `-57.41%`\n'
+                                        '*Sell Reason:* `stop_loss`\n'
+                                        '*Duration:* `2:35:03 (155.1 min)`\n'
                                         '*Amount:* `1333.33333333`\n'
                                         '*Open Rate:* `0.00007500`\n'
                                         '*Current Rate:* `0.00003201`\n'
-                                        '*Close Rate:* `0.00003201`\n'
-                                        '*Sell Reason:* `stop_loss`\n'
-                                        '*Duration:* `2:35:03 (155.1 min)`\n'
-                                        '*Profit:* `-57.41%`')
+                                        '*Close Rate:* `0.00003201`'
+                                        )
 
 
 @pytest.mark.parametrize('msg,expected', [
@@ -1482,7 +1600,7 @@ def test__send_msg_keyboard(default_conf, mocker, caplog) -> None:
                          ['/count', '/start', '/stop', '/help']]
     default_keyboard = ReplyKeyboardMarkup(default_keys_list)
 
-    custom_keys_list = [['/daily', '/stats', '/balance', '/profit'],
+    custom_keys_list = [['/daily', '/stats', '/balance', '/profit', '/profit 5'],
                         ['/count', '/start', '/reload_config', '/help']]
     custom_keyboard = ReplyKeyboardMarkup(custom_keys_list)
 
@@ -1516,5 +1634,5 @@ def test__send_msg_keyboard(default_conf, mocker, caplog) -> None:
     used_keyboard = bot.send_message.call_args[1]['reply_markup']
     assert used_keyboard == custom_keyboard
     assert log_has("using custom keyboard from config.json: "
-                   "[['/daily', '/stats', '/balance', '/profit'], ['/count', "
+                   "[['/daily', '/stats', '/balance', '/profit', '/profit 5'], ['/count', "
                    "'/start', '/reload_config', '/help']]", caplog)
diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py
index 4ca547390..04e63a3be 100644
--- a/tests/rpc/test_rpc_webhook.py
+++ b/tests/rpc/test_rpc_webhook.py
@@ -5,9 +5,9 @@ from unittest.mock import MagicMock
 import pytest
 from requests import RequestException
 
-from freqtrade.rpc import RPC, RPCMessageType
+from freqtrade.enums import RPCMessageType, SellType
+from freqtrade.rpc import RPC
 from freqtrade.rpc.webhook import Webhook
-from freqtrade.strategy.interface import SellType
 from tests.conftest import get_patched_freqtradebot, log_has
 
 
@@ -25,6 +25,11 @@ def get_webhook_dict() -> dict:
             "value2": "limit {limit:8f}",
             "value3": "{stake_amount:8f} {stake_currency}"
         },
+        "webhookbuyfill": {
+            "value1": "Buy Order for {pair} filled",
+            "value2": "at {open_rate:8f}",
+            "value3": "{stake_amount:8f} {stake_currency}"
+        },
         "webhooksell": {
             "value1": "Selling {pair}",
             "value2": "limit {limit:8f}",
@@ -35,6 +40,11 @@ def get_webhook_dict() -> dict:
             "value2": "limit {limit:8f}",
             "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})"
         },
+        "webhooksellfill": {
+            "value1": "Sell Order for {pair} filled",
+            "value2": "at {close_rate:8f}",
+            "value3": ""
+        },
         "webhookstatus": {
             "value1": "Status: {status}",
             "value2": "",
@@ -49,7 +59,7 @@ def test__init__(mocker, default_conf):
     assert webhook._config == default_conf
 
 
-def test_send_msg(default_conf, mocker):
+def test_send_msg_webhook(default_conf, mocker):
     default_conf["webhook"] = get_webhook_dict()
     msg_mock = MagicMock()
     mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
@@ -58,8 +68,8 @@ def test_send_msg(default_conf, mocker):
     msg_mock = MagicMock()
     mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
     msg = {
-        'type': RPCMessageType.BUY_NOTIFICATION,
-        'exchange': 'Bittrex',
+        'type': RPCMessageType.BUY,
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'limit': 0.005,
         'stake_amount': 0.8,
@@ -76,11 +86,11 @@ def test_send_msg(default_conf, mocker):
     assert (msg_mock.call_args[0][0]["value3"] ==
             default_conf["webhook"]["webhookbuy"]["value3"].format(**msg))
     # Test buy cancel
-    msg_mock = MagicMock()
-    mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
+    msg_mock.reset_mock()
+
     msg = {
-        'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
-        'exchange': 'Bittrex',
+        'type': RPCMessageType.BUY_CANCEL,
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'limit': 0.005,
         'stake_amount': 0.8,
@@ -96,12 +106,32 @@ def test_send_msg(default_conf, mocker):
             default_conf["webhook"]["webhookbuycancel"]["value2"].format(**msg))
     assert (msg_mock.call_args[0][0]["value3"] ==
             default_conf["webhook"]["webhookbuycancel"]["value3"].format(**msg))
-    # Test sell
-    msg_mock = MagicMock()
-    mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
+    # Test buy fill
+    msg_mock.reset_mock()
+
     msg = {
-        'type': RPCMessageType.SELL_NOTIFICATION,
-        'exchange': 'Bittrex',
+        'type': RPCMessageType.BUY_FILL,
+        'exchange': 'Binance',
+        'pair': 'ETH/BTC',
+        'open_rate': 0.005,
+        'stake_amount': 0.8,
+        'stake_amount_fiat': 500,
+        'stake_currency': 'BTC',
+        'fiat_currency': 'EUR'
+    }
+    webhook.send_msg(msg=msg)
+    assert msg_mock.call_count == 1
+    assert (msg_mock.call_args[0][0]["value1"] ==
+            default_conf["webhook"]["webhookbuyfill"]["value1"].format(**msg))
+    assert (msg_mock.call_args[0][0]["value2"] ==
+            default_conf["webhook"]["webhookbuyfill"]["value2"].format(**msg))
+    assert (msg_mock.call_args[0][0]["value3"] ==
+            default_conf["webhook"]["webhookbuyfill"]["value3"].format(**msg))
+    # Test sell
+    msg_mock.reset_mock()
+    msg = {
+        'type': RPCMessageType.SELL,
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'gain': "profit",
         'limit': 0.005,
@@ -123,11 +153,10 @@ def test_send_msg(default_conf, mocker):
     assert (msg_mock.call_args[0][0]["value3"] ==
             default_conf["webhook"]["webhooksell"]["value3"].format(**msg))
     # Test sell cancel
-    msg_mock = MagicMock()
-    mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
+    msg_mock.reset_mock()
     msg = {
-        'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
-        'exchange': 'Bittrex',
+        'type': RPCMessageType.SELL_CANCEL,
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'gain': "profit",
         'limit': 0.005,
@@ -148,9 +177,35 @@ def test_send_msg(default_conf, mocker):
             default_conf["webhook"]["webhooksellcancel"]["value2"].format(**msg))
     assert (msg_mock.call_args[0][0]["value3"] ==
             default_conf["webhook"]["webhooksellcancel"]["value3"].format(**msg))
-    for msgtype in [RPCMessageType.STATUS_NOTIFICATION,
-                    RPCMessageType.WARNING_NOTIFICATION,
-                    RPCMessageType.STARTUP_NOTIFICATION]:
+    # Test Sell fill
+    msg_mock.reset_mock()
+    msg = {
+        'type': RPCMessageType.SELL_FILL,
+        'exchange': 'Binance',
+        'pair': 'ETH/BTC',
+        'gain': "profit",
+        'close_rate': 0.005,
+        'amount': 0.8,
+        'order_type': 'limit',
+        'open_rate': 0.004,
+        'current_rate': 0.005,
+        'profit_amount': 0.001,
+        'profit_ratio': 0.20,
+        'stake_currency': 'BTC',
+        'sell_reason': SellType.STOP_LOSS.value
+    }
+    webhook.send_msg(msg=msg)
+    assert msg_mock.call_count == 1
+    assert (msg_mock.call_args[0][0]["value1"] ==
+            default_conf["webhook"]["webhooksellfill"]["value1"].format(**msg))
+    assert (msg_mock.call_args[0][0]["value2"] ==
+            default_conf["webhook"]["webhooksellfill"]["value2"].format(**msg))
+    assert (msg_mock.call_args[0][0]["value3"] ==
+            default_conf["webhook"]["webhooksellfill"]["value3"].format(**msg))
+
+    for msgtype in [RPCMessageType.STATUS,
+                    RPCMessageType.WARNING,
+                    RPCMessageType.STARTUP]:
         # Test notification
         msg = {
             'type': msgtype,
@@ -173,8 +228,8 @@ def test_exception_send_msg(default_conf, mocker, caplog):
     del default_conf["webhook"]["webhookbuy"]
 
     webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf)
-    webhook.send_msg({'type': RPCMessageType.BUY_NOTIFICATION})
-    assert log_has(f"Message type '{RPCMessageType.BUY_NOTIFICATION}' not configured for webhooks",
+    webhook.send_msg({'type': RPCMessageType.BUY})
+    assert log_has(f"Message type '{RPCMessageType.BUY}' not configured for webhooks",
                    caplog)
 
     default_conf["webhook"] = get_webhook_dict()
@@ -183,8 +238,8 @@ def test_exception_send_msg(default_conf, mocker, caplog):
     mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
     webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf)
     msg = {
-        'type': RPCMessageType.BUY_NOTIFICATION,
-        'exchange': 'Bittrex',
+        'type': RPCMessageType.BUY,
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'limit': 0.005,
         'order_type': 'limit',
@@ -225,3 +280,15 @@ def test__send_msg(default_conf, mocker, caplog):
     mocker.patch("freqtrade.rpc.webhook.post", post)
     webhook._send_msg(msg)
     assert log_has('Could not call webhook url. Exception: ', caplog)
+
+
+def test__send_msg_with_json_format(default_conf, mocker, caplog):
+    default_conf["webhook"] = get_webhook_dict()
+    default_conf["webhook"]["format"] = "json"
+    webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf)
+    msg = {'text': 'Hello'}
+    post = MagicMock()
+    mocker.patch("freqtrade.rpc.webhook.post", post)
+    webhook._send_msg(msg)
+
+    assert post.call_args[1] == {'json': msg}
diff --git a/tests/strategy/strats/default_strategy.py b/tests/strategy/strats/default_strategy.py
index 98842ff7c..7171b93ae 100644
--- a/tests/strategy/strats/default_strategy.py
+++ b/tests/strategy/strats/default_strategy.py
@@ -28,7 +28,7 @@ class DefaultStrategy(IStrategy):
     # Optimal stoploss designed for the strategy
     stoploss = -0.10
 
-    # Optimal ticker interval for the strategy
+    # Optimal timeframe for the strategy
     timeframe = '5m'
 
     # Optional order type mapping
diff --git a/tests/strategy/strats/hyperoptable_strategy.py b/tests/strategy/strats/hyperoptable_strategy.py
new file mode 100644
index 000000000..cc4734e13
--- /dev/null
+++ b/tests/strategy/strats/hyperoptable_strategy.py
@@ -0,0 +1,173 @@
+# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
+
+import talib.abstract as ta
+from pandas import DataFrame
+
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+from freqtrade.strategy import DecimalParameter, IntParameter, IStrategy, RealParameter
+
+
+class HyperoptableStrategy(IStrategy):
+    """
+    Default Strategy provided by freqtrade bot.
+    Please do not modify this strategy, it's  intended for internal use only.
+    Please look at the SampleStrategy in the user_data/strategy directory
+    or strategy repository https://github.com/freqtrade/freqtrade-strategies
+    for samples and inspiration.
+    """
+    INTERFACE_VERSION = 2
+
+    # Minimal ROI designed for the strategy
+    minimal_roi = {
+        "40": 0.0,
+        "30": 0.01,
+        "20": 0.02,
+        "0": 0.04
+    }
+
+    # Optimal stoploss designed for the strategy
+    stoploss = -0.10
+
+    # Optimal ticker interval for the strategy
+    timeframe = '5m'
+
+    # Optional order type mapping
+    order_types = {
+        'buy': 'limit',
+        'sell': 'limit',
+        'stoploss': 'limit',
+        'stoploss_on_exchange': False
+    }
+
+    # Number of candles the strategy requires before producing valid signals
+    startup_candle_count: int = 20
+
+    # Optional time in force for orders
+    order_time_in_force = {
+        'buy': 'gtc',
+        'sell': 'gtc',
+    }
+
+    buy_params = {
+        'buy_rsi': 35,
+        # Intentionally not specified, so "default" is tested
+        # 'buy_plusdi': 0.4
+    }
+
+    sell_params = {
+        'sell_rsi': 74,
+        'sell_minusdi': 0.4
+    }
+
+    buy_rsi = IntParameter([0, 50], default=30, space='buy')
+    buy_plusdi = RealParameter(low=0, high=1, default=0.5, space='buy')
+    sell_rsi = IntParameter(low=50, high=100, default=70, space='sell')
+    sell_minusdi = DecimalParameter(low=0, high=1, default=0.5001, decimals=3, space='sell',
+                                    load=False)
+
+    def informative_pairs(self):
+        """
+        Define additional, informative pair/interval combinations to be cached from the exchange.
+        These pair/interval combinations are non-tradeable, unless they are part
+        of the whitelist as well.
+        For more information, please consult the documentation
+        :return: List of tuples in the format (pair, interval)
+            Sample: return [("ETH/USDT", "5m"),
+                            ("BTC/USDT", "15m"),
+                            ]
+        """
+        return []
+
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        """
+        Adds several different TA indicators to the given DataFrame
+
+        Performance Note: For the best performance be frugal on the number of indicators
+        you are using. Let uncomment only the indicator you are using in your strategies
+        or your hyperopt configuration, otherwise you will waste your memory and CPU usage.
+        :param dataframe: Dataframe with data from the exchange
+        :param metadata: Additional information, like the currently traded pair
+        :return: a Dataframe with all mandatory indicators for the strategies
+        """
+
+        # Momentum Indicator
+        # ------------------------------------
+
+        # ADX
+        dataframe['adx'] = ta.ADX(dataframe)
+
+        # MACD
+        macd = ta.MACD(dataframe)
+        dataframe['macd'] = macd['macd']
+        dataframe['macdsignal'] = macd['macdsignal']
+        dataframe['macdhist'] = macd['macdhist']
+
+        # Minus Directional Indicator / Movement
+        dataframe['minus_di'] = ta.MINUS_DI(dataframe)
+
+        # Plus Directional Indicator / Movement
+        dataframe['plus_di'] = ta.PLUS_DI(dataframe)
+
+        # RSI
+        dataframe['rsi'] = ta.RSI(dataframe)
+
+        # Stoch fast
+        stoch_fast = ta.STOCHF(dataframe)
+        dataframe['fastd'] = stoch_fast['fastd']
+        dataframe['fastk'] = stoch_fast['fastk']
+
+        # Bollinger bands
+        bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2)
+        dataframe['bb_lowerband'] = bollinger['lower']
+        dataframe['bb_middleband'] = bollinger['mid']
+        dataframe['bb_upperband'] = bollinger['upper']
+
+        # EMA - Exponential Moving Average
+        dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10)
+
+        return dataframe
+
+    def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        """
+        Based on TA indicators, populates the buy signal for the given dataframe
+        :param dataframe: DataFrame
+        :param metadata: Additional information, like the currently traded pair
+        :return: DataFrame with buy column
+        """
+        dataframe.loc[
+            (
+                (dataframe['rsi'] < self.buy_rsi.value) &
+                (dataframe['fastd'] < 35) &
+                (dataframe['adx'] > 30) &
+                (dataframe['plus_di'] > self.buy_plusdi.value)
+            ) |
+            (
+                (dataframe['adx'] > 65) &
+                (dataframe['plus_di'] > self.buy_plusdi.value)
+            ),
+            'buy'] = 1
+
+        return dataframe
+
+    def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        """
+        Based on TA indicators, populates the sell signal for the given dataframe
+        :param dataframe: DataFrame
+        :param metadata: Additional information, like the currently traded pair
+        :return: DataFrame with buy column
+        """
+        dataframe.loc[
+            (
+                (
+                    (qtpylib.crossed_above(dataframe['rsi'], self.sell_rsi.value)) |
+                    (qtpylib.crossed_above(dataframe['fastd'], 70))
+                ) &
+                (dataframe['adx'] > 10) &
+                (dataframe['minus_di'] > 0)
+            ) |
+            (
+                (dataframe['adx'] > 70) &
+                (dataframe['minus_di'] > self.sell_minusdi.value)
+            ),
+            'sell'] = 1
+        return dataframe
diff --git a/tests/strategy/strats/legacy_strategy.py b/tests/strategy/strats/legacy_strategy.py
index 1e7bb5e1e..9ef00b110 100644
--- a/tests/strategy/strats/legacy_strategy.py
+++ b/tests/strategy/strats/legacy_strategy.py
@@ -31,7 +31,7 @@ class TestStrategyLegacy(IStrategy):
     # This attribute will be overridden if the config file contains "stoploss"
     stoploss = -0.10
 
-    # Optimal ticker interval for the strategy
+    # Optimal timeframe for the strategy
     # Keep the legacy value here to test compatibility
     ticker_interval = '5m'
 
diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py
index ec7b3c33d..92ac9f63a 100644
--- a/tests/strategy/test_default_strategy.py
+++ b/tests/strategy/test_default_strategy.py
@@ -36,9 +36,11 @@ def test_default_strategy(result, fee):
     )
 
     assert strategy.confirm_trade_entry(pair='ETH/BTC', order_type='limit', amount=0.1,
-                                        rate=20000, time_in_force='gtc') is True
+                                        rate=20000, time_in_force='gtc',
+                                        current_time=datetime.utcnow()) is True
     assert strategy.confirm_trade_exit(pair='ETH/BTC', trade=trade, order_type='limit', amount=0.1,
-                                       rate=20000, time_in_force='gtc', sell_reason='roi') is True
+                                       rate=20000, time_in_force='gtc', sell_reason='roi',
+                                       current_time=datetime.utcnow()) is True
 
     assert strategy.custom_stoploss(pair='ETH/BTC', trade=trade, current_time=datetime.now(),
                                     current_rate=20_000, current_profit=0.05) == strategy.stoploss
diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py
index f158a1518..64081fa37 100644
--- a/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@@ -10,10 +10,13 @@ from pandas import DataFrame
 from freqtrade.configuration import TimeRange
 from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history import load_data
-from freqtrade.exceptions import StrategyError
+from freqtrade.enums import SellType
+from freqtrade.exceptions import OperationalException, StrategyError
 from freqtrade.persistence import PairLocks, Trade
 from freqtrade.resolvers import StrategyResolver
-from freqtrade.strategy.interface import SellCheckTuple, SellType
+from freqtrade.strategy.hyper import (BaseParameter, CategoricalParameter, DecimalParameter,
+                                      IntParameter, RealParameter)
+from freqtrade.strategy.interface import SellCheckTuple
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from tests.conftest import log_has, log_has_re
 
@@ -217,7 +220,7 @@ def test_min_roi_reached(default_conf, fee) -> None:
             open_date=arrow.utcnow().shift(hours=-1).datetime,
             fee_open=fee.return_value,
             fee_close=fee.return_value,
-            exchange='bittrex',
+            exchange='binance',
             open_rate=1,
         )
 
@@ -256,7 +259,7 @@ def test_min_roi_reached2(default_conf, fee) -> None:
             open_date=arrow.utcnow().shift(hours=-1).datetime,
             fee_open=fee.return_value,
             fee_close=fee.return_value,
-            exchange='bittrex',
+            exchange='binance',
             open_rate=1,
         )
 
@@ -291,7 +294,7 @@ def test_min_roi_reached3(default_conf, fee) -> None:
         open_date=arrow.utcnow().shift(hours=-1).datetime,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
         open_rate=1,
     )
 
@@ -344,7 +347,7 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili
         open_date=arrow.utcnow().shift(hours=-1).datetime,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
         open_rate=1,
     )
     trade.adjust_min_max_rates(trade.open_rate)
@@ -380,6 +383,50 @@ def test_stop_loss_reached(default_conf, fee, profit, adjusted, expected, traili
     strategy.custom_stoploss = original_stopvalue
 
 
+def test_custom_sell(default_conf, fee, caplog) -> None:
+
+    default_conf.update({'strategy': 'DefaultStrategy'})
+
+    strategy = StrategyResolver.load_strategy(default_conf)
+    trade = Trade(
+        pair='ETH/BTC',
+        stake_amount=0.01,
+        amount=1,
+        open_date=arrow.utcnow().shift(hours=-1).datetime,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        exchange='binance',
+        open_rate=1,
+    )
+
+    now = arrow.utcnow().datetime
+    res = strategy.should_sell(trade, 1, now, False, False, None, None, 0)
+
+    assert res.sell_flag is False
+    assert res.sell_type == SellType.NONE
+
+    strategy.custom_sell = MagicMock(return_value=True)
+    res = strategy.should_sell(trade, 1, now, False, False, None, None, 0)
+    assert res.sell_flag is True
+    assert res.sell_type == SellType.CUSTOM_SELL
+    assert res.sell_reason == 'custom_sell'
+
+    strategy.custom_sell = MagicMock(return_value='hello world')
+
+    res = strategy.should_sell(trade, 1, now, False, False, None, None, 0)
+    assert res.sell_type == SellType.CUSTOM_SELL
+    assert res.sell_flag is True
+    assert res.sell_reason == 'hello world'
+
+    caplog.clear()
+    strategy.custom_sell = MagicMock(return_value='h' * 100)
+    res = strategy.should_sell(trade, 1, now, False, False, None, None, 0)
+    assert res.sell_type == SellType.CUSTOM_SELL
+    assert res.sell_flag is True
+    assert res.sell_reason == 'h' * 64
+    assert log_has_re('Custom sell reason returned from custom_sell is too long.*', caplog)
+
+
 def test_analyze_ticker_default(ohlcv_history, mocker, caplog) -> None:
     caplog.set_level(logging.DEBUG)
     ind_mock = MagicMock(side_effect=lambda x, meta: x)
@@ -552,3 +599,82 @@ def test_strategy_safe_wrapper(value):
 
     assert type(ret) == type(value)
     assert ret == value
+
+
+def test_hyperopt_parameters():
+    from skopt.space import Categorical, Integer, Real
+    with pytest.raises(OperationalException, match=r"Name is determined.*"):
+        IntParameter(low=0, high=5, default=1, name='hello')
+
+    with pytest.raises(OperationalException, match=r"IntParameter space must be.*"):
+        IntParameter(low=0, default=5, space='buy')
+
+    with pytest.raises(OperationalException, match=r"RealParameter space must be.*"):
+        RealParameter(low=0, default=5, space='buy')
+
+    with pytest.raises(OperationalException, match=r"DecimalParameter space must be.*"):
+        DecimalParameter(low=0, default=5, space='buy')
+
+    with pytest.raises(OperationalException, match=r"IntParameter space invalid\."):
+        IntParameter([0, 10], high=7, default=5, space='buy')
+
+    with pytest.raises(OperationalException, match=r"RealParameter space invalid\."):
+        RealParameter([0, 10], high=7, default=5, space='buy')
+
+    with pytest.raises(OperationalException, match=r"DecimalParameter space invalid\."):
+        DecimalParameter([0, 10], high=7, default=5, space='buy')
+
+    with pytest.raises(OperationalException, match=r"CategoricalParameter space must.*"):
+        CategoricalParameter(['aa'], default='aa', space='buy')
+
+    with pytest.raises(TypeError):
+        BaseParameter(opt_range=[0, 1], default=1, space='buy')
+
+    intpar = IntParameter(low=0, high=5, default=1, space='buy')
+    assert intpar.value == 1
+    assert isinstance(intpar.get_space(''), Integer)
+    assert isinstance(intpar.range, range)
+    assert len(list(intpar.range)) == 1
+    # Range contains ONLY the default / value.
+    assert list(intpar.range) == [intpar.value]
+    intpar.in_space = True
+
+    assert len(list(intpar.range)) == 6
+    assert list(intpar.range) == [0, 1, 2, 3, 4, 5]
+
+    fltpar = RealParameter(low=0.0, high=5.5, default=1.0, space='buy')
+    assert isinstance(fltpar.get_space(''), Real)
+    assert fltpar.value == 1
+
+    fltpar = DecimalParameter(low=0.0, high=5.5, default=1.0004, decimals=3, space='buy')
+    assert isinstance(fltpar.get_space(''), Integer)
+    assert fltpar.value == 1
+
+    catpar = CategoricalParameter(['buy_rsi', 'buy_macd', 'buy_none'],
+                                  default='buy_macd', space='buy')
+    assert isinstance(catpar.get_space(''), Categorical)
+    assert catpar.value == 'buy_macd'
+
+
+def test_auto_hyperopt_interface(default_conf):
+    default_conf.update({'strategy': 'HyperoptableStrategy'})
+    PairLocks.timeframe = default_conf['timeframe']
+    strategy = StrategyResolver.load_strategy(default_conf)
+
+    assert strategy.buy_rsi.value == strategy.buy_params['buy_rsi']
+    # PlusDI is NOT in the buy-params, so default should be used
+    assert strategy.buy_plusdi.value == 0.5
+    assert strategy.sell_rsi.value == strategy.sell_params['sell_rsi']
+
+    # Parameter is disabled - so value from sell_param dict will NOT be used.
+    assert strategy.sell_minusdi.value == 0.5
+    all_params = strategy.detect_all_parameters()
+    assert isinstance(all_params, dict)
+    assert len(all_params['buy']) == 2
+    assert len(all_params['sell']) == 2
+    assert all_params['count'] == 4
+
+    strategy.__class__.sell_rsi = IntParameter([0, 10], default=5, space='buy')
+
+    with pytest.raises(OperationalException, match=r"Inconclusive parameter.*"):
+        [x for x in strategy.detect_parameters('sell')]
diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py
index 252288e2e..3b84fc254 100644
--- a/tests/strategy/test_strategy_helpers.py
+++ b/tests/strategy/test_strategy_helpers.py
@@ -1,8 +1,10 @@
+from math import isclose
+
 import numpy as np
 import pandas as pd
 import pytest
 
-from freqtrade.strategy import merge_informative_pair, timeframe_to_minutes
+from freqtrade.strategy import merge_informative_pair, stoploss_from_open, timeframe_to_minutes
 
 
 def generate_test_data(timeframe: str, size: int):
@@ -95,3 +97,38 @@ def test_merge_informative_pair_lower():
 
     with pytest.raises(ValueError, match=r"Tried to merge a faster timeframe .*"):
         merge_informative_pair(data, informative, '1h', '15m', ffill=True)
+
+
+def test_stoploss_from_open():
+    open_price_ranges = [
+        [0.01, 1.00, 30],
+        [1, 100, 30],
+        [100, 10000, 30],
+    ]
+    current_profit_range = [-0.99, 2, 30]
+    desired_stop_range = [-0.50, 0.50, 30]
+
+    for open_range in open_price_ranges:
+        for open_price in np.linspace(*open_range):
+            for desired_stop in np.linspace(*desired_stop_range):
+
+                # -1 is not a valid current_profit, should return 1
+                assert stoploss_from_open(desired_stop, -1) == 1
+
+                for current_profit in np.linspace(*current_profit_range):
+                    current_price = open_price * (1 + current_profit)
+                    expected_stop_price = open_price * (1 + desired_stop)
+
+                    stoploss = stoploss_from_open(desired_stop, current_profit)
+
+                    assert stoploss >= 0
+                    assert stoploss <= 1
+
+                    stop_price = current_price * (1 - stoploss)
+
+                    # there is no correct answer if the expected stop price is above
+                    # the current price
+                    if expected_stop_price > current_price:
+                        assert stoploss == 0
+                    else:
+                        assert isclose(stop_price, expected_stop_price, rel_tol=0.00001)
diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py
index 1c692d2da..bd192ecb5 100644
--- a/tests/strategy/test_strategy_loading.py
+++ b/tests/strategy/test_strategy_loading.py
@@ -35,7 +35,7 @@ def test_search_all_strategies_no_failed():
     directory = Path(__file__).parent / "strats"
     strategies = StrategyResolver.search_all_objects(directory, enum_failed=False)
     assert isinstance(strategies, list)
-    assert len(strategies) == 2
+    assert len(strategies) == 3
     assert isinstance(strategies[0], dict)
 
 
@@ -43,10 +43,10 @@ def test_search_all_strategies_with_failed():
     directory = Path(__file__).parent / "strats"
     strategies = StrategyResolver.search_all_objects(directory, enum_failed=True)
     assert isinstance(strategies, list)
-    assert len(strategies) == 3
+    assert len(strategies) == 4
     # with enum_failed=True search_all_objects() shall find 2 good strategies
     # and 1 which fails to load
-    assert len([x for x in strategies if x['class'] is not None]) == 2
+    assert len([x for x in strategies if x['class'] is not None]) == 3
     assert len([x for x in strategies if x['class'] is None]) == 1
 
 
@@ -129,13 +129,16 @@ def test_strategy_override_minimal_roi(caplog, default_conf):
     default_conf.update({
         'strategy': 'DefaultStrategy',
         'minimal_roi': {
+            "20": 0.1,
             "0": 0.5
         }
     })
     strategy = StrategyResolver.load_strategy(default_conf)
 
     assert strategy.minimal_roi[0] == 0.5
-    assert log_has("Override strategy 'minimal_roi' with value in config file: {'0': 0.5}.", caplog)
+    assert log_has(
+        "Override strategy 'minimal_roi' with value in config file: {'20': 0.1, '0': 0.5}.",
+        caplog)
 
 
 def test_strategy_override_stoploss(caplog, default_conf):
diff --git a/tests/test_arguments.py b/tests/test_arguments.py
index 60c2cfbac..0d81dea28 100644
--- a/tests/test_arguments.py
+++ b/tests/test_arguments.py
@@ -186,18 +186,22 @@ def test_plot_dataframe_options() -> None:
     assert pargs['pairs'] == ['UNITTEST/BTC']
 
 
-def test_plot_profit_options() -> None:
+@pytest.mark.parametrize('auto_open_arg', [True, False])
+def test_plot_profit_options(auto_open_arg: bool) -> None:
     args = [
         'plot-profit',
         '-p', 'UNITTEST/BTC',
         '--trade-source', 'DB',
         '--db-url', 'sqlite:///whatever.sqlite',
     ]
+    if auto_open_arg:
+        args.append('--auto-open')
     pargs = Arguments(args).get_parsed_arg()
 
     assert pargs['trade_source'] == 'DB'
     assert pargs['pairs'] == ['UNITTEST/BTC']
     assert pargs['db_url'] == 'sqlite:///whatever.sqlite'
+    assert pargs['plot_auto_open'] == auto_open_arg
 
 
 def test_config_notallowed(mocker) -> None:
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index 94c3e24f6..aa121edfa 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -18,11 +18,11 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti
                                                          process_deprecated_setting,
                                                          process_removed_setting,
                                                          process_temporary_deprecated_settings)
-from freqtrade.configuration.load_config import load_config_file, log_config_error_range
+from freqtrade.configuration.load_config import load_config_file, load_file, log_config_error_range
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
+from freqtrade.enums import RunMode
 from freqtrade.exceptions import OperationalException
 from freqtrade.loggers import _set_loggers, setup_logging, setup_logging_pre
-from freqtrade.state import RunMode
 from tests.conftest import log_has, log_has_re, patched_configuration_load_config_file
 
 
@@ -88,6 +88,24 @@ def test_load_config_file_error_range(default_conf, mocker, caplog) -> None:
             '"stake_amount": .001, "fiat_display_currency": "USD", '
             '"timeframe": "5m", "dry_run": true, "cance')
 
+    filedata = json.dumps(default_conf, indent=2).replace(
+        '"stake_amount": 0.001,', '"stake_amount": .001,')
+    mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata))
+
+    x = log_config_error_range('somefile', 'Parse error at offset 4: Invalid value.')
+    assert isinstance(x, str)
+    assert (x == '  "max_open_trades": 1,\n  "stake_currency": "BTC",\n'
+            '  "stake_amount": .001,')
+
+    x = log_config_error_range('-', '')
+    assert x == ''
+
+
+def test_load_file_error(tmpdir):
+    testpath = Path(tmpdir) / 'config.json'
+    with pytest.raises(OperationalException, match=r"File .* not found!"):
+        load_file(testpath)
+
 
 def test__args_to_config(caplog):
 
@@ -430,7 +448,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
         '--enable-position-stacking',
         '--disable-max-market-positions',
         '--timerange', ':100',
-        '--export', '/bar/foo'
+        '--export', '/bar/foo',
+        '--stake-amount', 'unlimited'
     ]
 
     args = Arguments(arglist).get_parsed_arg()
@@ -463,6 +482,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
 
     assert 'export' in config
     assert log_has('Parameter --export detected: {} ...'.format(config['export']), caplog)
+    assert 'stake_amount' in config
+    assert config['stake_amount'] == 'unlimited'
 
 
 def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> None:
@@ -562,7 +583,7 @@ def test_check_exchange(default_conf, caplog) -> None:
     # Test a 'bad' exchange, which known to have serious problems
     default_conf.get('exchange').update({'name': 'bitmex'})
     with pytest.raises(OperationalException,
-                       match=r"Exchange .* is known to not work with the bot yet.*"):
+                       match=r"Exchange .* will not work with Freqtrade\..*"):
         check_exchange(default_conf)
     caplog.clear()
 
@@ -787,6 +808,38 @@ def test_validate_max_open_trades(default_conf):
         validate_config_consistency(default_conf)
 
 
+def test_validate_price_side(default_conf):
+    default_conf['order_types'] = {
+        "buy": "limit",
+        "sell": "limit",
+        "stoploss": "limit",
+        "stoploss_on_exchange": False,
+    }
+    # Default should pass
+    validate_config_consistency(default_conf)
+
+    conf = deepcopy(default_conf)
+    conf['order_types']['buy'] = 'market'
+    with pytest.raises(OperationalException,
+                       match='Market buy orders require bid_strategy.price_side = "ask".'):
+        validate_config_consistency(conf)
+
+    conf = deepcopy(default_conf)
+    conf['order_types']['sell'] = 'market'
+    with pytest.raises(OperationalException,
+                       match='Market sell orders require ask_strategy.price_side = "bid".'):
+        validate_config_consistency(conf)
+
+    # Validate inversed case
+    conf = deepcopy(default_conf)
+    conf['order_types']['sell'] = 'market'
+    conf['order_types']['buy'] = 'market'
+    conf['ask_strategy']['price_side'] = 'bid'
+    conf['bid_strategy']['price_side'] = 'ask'
+
+    validate_config_consistency(conf)
+
+
 def test_validate_tsl(default_conf):
     default_conf['stoploss'] = 0.0
     with pytest.raises(OperationalException, match='The config stoploss needs to be different '
@@ -825,22 +878,6 @@ def test_validate_tsl(default_conf):
         validate_config_consistency(default_conf)
 
 
-def test_validate_edge(edge_conf):
-    edge_conf.update({"pairlist": {
-        "method": "VolumePairList",
-    }})
-
-    with pytest.raises(OperationalException,
-                       match="Edge and VolumePairList are incompatible, "
-                       "Edge will override whatever pairs VolumePairlist selects."):
-        validate_config_consistency(edge_conf)
-
-    edge_conf.update({"pairlist": {
-        "method": "StaticPairList",
-    }})
-    validate_config_consistency(edge_conf)
-
-
 def test_validate_edge2(edge_conf):
     edge_conf.update({"ask_strategy": {
         "use_sell_signal": True,
@@ -983,6 +1020,7 @@ def test_pairlist_resolving():
     config = configuration.get_config()
 
     assert config['pairs'] == ['ETH/BTC', 'XRP/BTC']
+    assert config['exchange']['pair_whitelist'] == ['ETH/BTC', 'XRP/BTC']
     assert config['exchange']['name'] == 'binance'
 
 
@@ -1019,37 +1057,30 @@ def test_pairlist_resolving_with_config(mocker, default_conf):
 
 def test_pairlist_resolving_with_config_pl(mocker, default_conf):
     patched_configuration_load_config_file(mocker, default_conf)
-    load_mock = mocker.patch("freqtrade.configuration.configuration.json_load",
-                             MagicMock(return_value=['XRP/BTC', 'ETH/BTC']))
-    mocker.patch.object(Path, "exists", MagicMock(return_value=True))
-    mocker.patch.object(Path, "open", MagicMock(return_value=MagicMock()))
 
     arglist = [
         'download-data',
         '--config', 'config.json',
-        '--pairs-file', 'pairs.json',
+        '--pairs-file', 'tests/testdata/pairs.json',
     ]
 
     args = Arguments(arglist).get_parsed_arg()
 
     configuration = Configuration(args)
     config = configuration.get_config()
-
-    assert load_mock.call_count == 1
-    assert config['pairs'] == ['ETH/BTC', 'XRP/BTC']
+    assert len(config['pairs']) == 23
+    assert 'ETH/BTC' in config['pairs']
+    assert 'XRP/BTC' in config['pairs']
     assert config['exchange']['name'] == default_conf['exchange']['name']
 
 
 def test_pairlist_resolving_with_config_pl_not_exists(mocker, default_conf):
     patched_configuration_load_config_file(mocker, default_conf)
-    mocker.patch("freqtrade.configuration.configuration.json_load",
-                 MagicMock(return_value=['XRP/BTC', 'ETH/BTC']))
-    mocker.patch.object(Path, "exists", MagicMock(return_value=False))
 
     arglist = [
         'download-data',
         '--config', 'config.json',
-        '--pairs-file', 'pairs.json',
+        '--pairs-file', 'tests/testdata/pairs_doesnotexist.json',
     ]
 
     args = Arguments(arglist).get_parsed_arg()
@@ -1062,7 +1093,7 @@ def test_pairlist_resolving_with_config_pl_not_exists(mocker, default_conf):
 def test_pairlist_resolving_fallback(mocker):
     mocker.patch.object(Path, "exists", MagicMock(return_value=True))
     mocker.patch.object(Path, "open", MagicMock(return_value=MagicMock()))
-    mocker.patch("freqtrade.configuration.configuration.json_load",
+    mocker.patch("freqtrade.configuration.configuration.load_file",
                  MagicMock(return_value=['XRP/BTC', 'ETH/BTC']))
     arglist = [
         'download-data',
diff --git a/tests/test_directory_operations.py b/tests/test_directory_operations.py
index a8058c514..a11200526 100644
--- a/tests/test_directory_operations.py
+++ b/tests/test_directory_operations.py
@@ -1,11 +1,12 @@
 # pragma pylint: disable=missing-docstring, protected-access, invalid-name
+import os
 from pathlib import Path
 from unittest.mock import MagicMock
 
 import pytest
 
-from freqtrade.configuration.directory_operations import (copy_sample_files, create_datadir,
-                                                          create_userdata_dir)
+from freqtrade.configuration.directory_operations import (chown_user_directory, copy_sample_files,
+                                                          create_datadir, create_userdata_dir)
 from freqtrade.exceptions import OperationalException
 from tests.conftest import log_has, log_has_re
 
@@ -31,6 +32,24 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None:
     assert str(x) == str(Path("/tmp/bar"))
 
 
+def test_create_userdata_dir_and_chown(mocker, tmpdir, caplog) -> None:
+    sp_mock = mocker.patch('subprocess.check_output')
+    path = Path(tmpdir / 'bar')
+    assert not path.is_dir()
+
+    x = create_userdata_dir(str(path), create_dir=True)
+    assert sp_mock.call_count == 0
+    assert log_has(f'Created user-data directory: {path}', caplog)
+    assert isinstance(x, Path)
+    assert path.is_dir()
+    assert (path / 'data').is_dir()
+
+    os.environ['FT_APP_ENV'] = 'docker'
+    chown_user_directory(path / 'data')
+    assert sp_mock.call_count == 1
+    del os.environ['FT_APP_ENV']
+
+
 def test_create_userdata_dir_exists(mocker, default_conf, caplog) -> None:
     mocker.patch.object(Path, "is_dir", MagicMock(return_value=True))
     md = mocker.patch.object(Path, 'mkdir', MagicMock())
diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index 3bd2f5607..66f87f7c9 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -11,15 +11,14 @@ import arrow
 import pytest
 
 from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT
+from freqtrade.enums import RPCMessageType, RunMode, SellType, State
 from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, PricingError,
                                   TemporaryError)
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.persistence import Order, PairLocks, Trade
 from freqtrade.persistence.models import PairLock
-from freqtrade.rpc import RPCMessageType
-from freqtrade.state import RunMode, State
-from freqtrade.strategy.interface import SellCheckTuple, SellType
+from freqtrade.strategy.interface import SellCheckTuple
 from freqtrade.worker import Worker
 from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker,
                             log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal,
@@ -82,10 +81,6 @@ def test_bot_cleanup(mocker, default_conf, caplog) -> None:
 def test_order_dict_dry_run(default_conf, mocker, caplog) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2)
-    )
     conf = default_conf.copy()
     conf['runmode'] = RunMode.DRY_RUN
     conf['order_types'] = {
@@ -94,6 +89,7 @@ def test_order_dict_dry_run(default_conf, mocker, caplog) -> None:
         'stoploss': 'limit',
         'stoploss_on_exchange': True,
     }
+    conf['bid_strategy']['price_side'] = 'ask'
 
     freqtrade = FreqtradeBot(conf)
     assert freqtrade.strategy.order_types['stoploss_on_exchange']
@@ -116,10 +112,7 @@ def test_order_dict_dry_run(default_conf, mocker, caplog) -> None:
 def test_order_dict_live(default_conf, mocker, caplog) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2)
-    )
+
     conf = default_conf.copy()
     conf['runmode'] = RunMode.LIVE
     conf['order_types'] = {
@@ -128,6 +121,7 @@ def test_order_dict_live(default_conf, mocker, caplog) -> None:
         'stoploss': 'limit',
         'stoploss_on_exchange': True,
     }
+    conf['bid_strategy']['price_side'] = 'ask'
 
     freqtrade = FreqtradeBot(conf)
     assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog)
@@ -151,15 +145,10 @@ def test_order_dict_live(default_conf, mocker, caplog) -> None:
 def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2)
-    )
 
     freqtrade = FreqtradeBot(default_conf)
 
-    result = freqtrade.wallets.get_trade_stake_amount(
-        'ETH/BTC', freqtrade.get_free_open_trades())
+    result = freqtrade.wallets.get_trade_stake_amount('ETH/BTC')
     assert result == default_conf['stake_amount']
 
 
@@ -180,7 +169,6 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2),
         buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee
     )
@@ -195,73 +183,12 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b
 
         if expected[i] is not None:
             limit_buy_order_open['id'] = str(i)
-            result = freqtrade.wallets.get_trade_stake_amount('ETH/BTC',
-                                                              freqtrade.get_free_open_trades())
+            result = freqtrade.wallets.get_trade_stake_amount('ETH/BTC')
             assert pytest.approx(result) == expected[i]
             freqtrade.execute_buy('ETH/BTC', result)
         else:
             with pytest.raises(DependencyException):
-                freqtrade.wallets.get_trade_stake_amount('ETH/BTC',
-                                                         freqtrade.get_free_open_trades())
-
-
-def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None:
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
-    patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5)
-    freqtrade = FreqtradeBot(default_conf)
-    patch_get_signal(freqtrade)
-
-    with pytest.raises(DependencyException, match=r'.*stake amount.*'):
-        freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.get_free_open_trades())
-
-
-@pytest.mark.parametrize("balance_ratio,result1", [
-                        (1, 0.005),
-                        (0.99, 0.00495),
-                        (0.50, 0.0025),
-                        ])
-def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_ratio, result1,
-                                                 limit_buy_order_open, fee, mocker) -> None:
-    patch_RPCManager(mocker)
-    patch_exchange(mocker)
-    mocker.patch.multiple(
-        'freqtrade.exchange.Exchange',
-        fetch_ticker=ticker,
-        buy=MagicMock(return_value=limit_buy_order_open),
-        get_fee=fee
-    )
-
-    conf = deepcopy(default_conf)
-    conf['stake_amount'] = UNLIMITED_STAKE_AMOUNT
-    conf['dry_run_wallet'] = 0.01
-    conf['max_open_trades'] = 2
-    conf['tradable_balance_ratio'] = balance_ratio
-
-    freqtrade = FreqtradeBot(conf)
-    patch_get_signal(freqtrade)
-
-    # no open trades, order amount should be 'balance / max_open_trades'
-    result = freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.get_free_open_trades())
-    assert result == result1
-
-    # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)'
-    freqtrade.execute_buy('ETH/BTC', result)
-
-    result = freqtrade.wallets.get_trade_stake_amount('LTC/BTC', freqtrade.get_free_open_trades())
-    assert result == result1
-
-    # create 2 trades, order amount should be None
-    freqtrade.execute_buy('LTC/BTC', result)
-
-    result = freqtrade.wallets.get_trade_stake_amount('XRP/BTC', freqtrade.get_free_open_trades())
-    assert result == 0
-
-    # set max_open_trades = None, so do not trade
-    conf['max_open_trades'] = 0
-    freqtrade = FreqtradeBot(conf)
-    result = freqtrade.wallets.get_trade_stake_amount('NEO/BTC', freqtrade.get_free_open_trades())
-    assert result == 0
+                freqtrade.wallets.get_trade_stake_amount('ETH/BTC')
 
 
 def test_edge_called_in_process(mocker, edge_conf) -> None:
@@ -287,9 +214,9 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None:
     freqtrade = FreqtradeBot(edge_conf)
 
     assert freqtrade.wallets.get_trade_stake_amount(
-        'NEO/BTC', freqtrade.get_free_open_trades(), freqtrade.edge) == (999.9 * 0.5 * 0.01) / 0.20
+        'NEO/BTC', freqtrade.edge) == (999.9 * 0.5 * 0.01) / 0.20
     assert freqtrade.wallets.get_trade_stake_amount(
-        'LTC/BTC', freqtrade.get_free_open_trades(), freqtrade.edge) == (999.9 * 0.5 * 0.01) / 0.21
+        'LTC/BTC', freqtrade.edge) == (999.9 * 0.5 * 0.01) / 0.21
 
 
 def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf) -> None:
@@ -419,7 +346,7 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non
     assert trade.stake_amount == 0.001
     assert trade.is_open
     assert trade.open_date is not None
-    assert trade.exchange == 'bittrex'
+    assert trade.exchange == 'binance'
 
     # Simulate fulfilled LIMIT_BUY order for trade
     trade.update(limit_buy_order)
@@ -495,7 +422,6 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order_open,
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         buy=MagicMock(return_value=limit_buy_order_open),
-        get_balance=MagicMock(return_value=default_conf['stake_amount']),
         get_fee=fee,
     )
     default_conf['max_open_trades'] = 0
@@ -505,8 +431,7 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order_open,
     patch_get_signal(freqtrade)
 
     assert not freqtrade.create_trade('ETH/BTC')
-    assert freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.get_free_open_trades(),
-                                                    freqtrade.edge) == 0
+    assert freqtrade.wallets.get_trade_stake_amount('ETH/BTC', freqtrade.edge) == 0
 
 
 def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_open, fee,
@@ -584,7 +509,6 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
-        get_balance=MagicMock(return_value=20),
         get_fee=fee,
     )
     default_conf['stake_amount'] = 10
@@ -678,12 +602,13 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, limit_buy
     assert trade.stake_amount == default_conf['stake_amount']
     assert trade.is_open
     assert trade.open_date is not None
-    assert trade.exchange == 'bittrex'
+    assert trade.exchange == 'binance'
     assert trade.open_rate == 0.00001098
     assert trade.amount == 91.07468123
 
     assert log_has(
-        'Buy signal found: about create a new trade with stake_amount: 0.001 ...', caplog
+        'Buy signal found: about create a new trade for ETH/BTC with stake_amount: 0.001 ...',
+        caplog
     )
 
 
@@ -766,7 +691,7 @@ def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order,
     assert pair not in default_conf['exchange']['pair_whitelist']
 
     # create open trade not in whitelist
-    Trade.session.add(Trade(
+    Trade.query.session.add(Trade(
         pair=pair,
         stake_amount=0.001,
         fee_open=fee.return_value,
@@ -774,9 +699,9 @@ def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order,
         is_open=True,
         amount=20,
         open_rate=0.01,
-        exchange='bittrex',
+        exchange='binance',
     ))
-    Trade.session.add(Trade(
+    Trade.query.session.add(Trade(
         pair='ETH/BTC',
         stake_amount=0.001,
         fee_open=fee.return_value,
@@ -784,7 +709,7 @@ def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order,
         is_open=True,
         amount=12,
         open_rate=0.001,
-        exchange='bittrex',
+        exchange='binance',
     ))
 
     assert pair not in freqtrade.active_pair_whitelist
@@ -825,50 +750,6 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None:
     assert ("ETH/BTC", default_conf["timeframe"]) in refresh_mock.call_args[0][0]
 
 
-@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [
-    ('ask', 20, 19, 10, 0.0, 20),  # Full ask side
-    ('ask', 20, 19, 10, 1.0, 10),  # Full last side
-    ('ask', 20, 19, 10, 0.5, 15),  # Between ask and last
-    ('ask', 20, 19, 10, 0.7, 13),  # Between ask and last
-    ('ask', 20, 19, 10, 0.3, 17),  # Between ask and last
-    ('ask', 5, 6, 10, 1.0, 5),  # last bigger than ask
-    ('ask', 5, 6, 10, 0.5, 5),  # last bigger than ask
-    ('ask', 10, 20, None, 0.5, 10),  # last not available - uses ask
-    ('ask', 4, 5, None, 0.5, 4),  # last not available - uses ask
-    ('ask', 4, 5, None, 1, 4),  # last not available - uses ask
-    ('ask', 4, 5, None, 0, 4),  # last not available - uses ask
-    ('bid', 10, 20, 10, 0.0, 20),  # Full bid side
-    ('bid', 10, 20, 10, 1.0, 10),  # Full last side
-    ('bid', 10, 20, 10, 0.5, 15),  # Between bid and last
-    ('bid', 10, 20, 10, 0.7, 13),  # Between bid and last
-    ('bid', 10, 20, 10, 0.3, 17),  # Between bid and last
-    ('bid', 4, 5, 10, 1.0, 5),  # last bigger than bid
-    ('bid', 4, 5, 10, 0.5, 5),  # last bigger than bid
-    ('bid', 10, 20, None, 0.5, 20),  # last not available - uses bid
-    ('bid', 4, 5, None, 0.5, 5),  # last not available - uses bid
-    ('bid', 4, 5, None, 1, 5),  # last not available - uses bid
-    ('bid', 4, 5, None, 0, 5),  # last not available - uses bid
-])
-def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid,
-                      last, last_ab, expected) -> None:
-    caplog.set_level(logging.DEBUG)
-    default_conf['bid_strategy']['ask_last_balance'] = last_ab
-    default_conf['bid_strategy']['price_side'] = side
-    freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
-                 MagicMock(return_value={'ask': ask, 'last': last, 'bid': bid}))
-
-    assert freqtrade.get_buy_rate('ETH/BTC', True) == expected
-    assert not log_has("Using cached buy rate for ETH/BTC.", caplog)
-
-    assert freqtrade.get_buy_rate('ETH/BTC', False) == expected
-    assert log_has("Using cached buy rate for ETH/BTC.", caplog)
-    # Running a 2nd time with Refresh on!
-    caplog.clear()
-    assert freqtrade.get_buy_rate('ETH/BTC', True) == expected
-    assert not log_has("Using cached buy rate for ETH/BTC.", caplog)
-
-
 def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -877,13 +758,10 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order
     stake_amount = 2
     bid = 0.11
     buy_rate_mock = MagicMock(return_value=bid)
-    mocker.patch.multiple(
-        'freqtrade.freqtradebot.FreqtradeBot',
-        get_buy_rate=buy_rate_mock,
-    )
     buy_mm = MagicMock(return_value=limit_buy_order_open)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
+        get_buy_rate=buy_rate_mock,
         fetch_ticker=MagicMock(return_value={
             'bid': 0.00001172,
             'ask': 0.00001173,
@@ -974,7 +852,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order
     assert not freqtrade.execute_buy(pair, stake_amount)
 
     # Fail to get price...
-    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_buy_rate', MagicMock(return_value=0.0))
+    mocker.patch('freqtrade.exchange.Exchange.get_buy_rate', MagicMock(return_value=0.0))
 
     with pytest.raises(PricingError, match="Could not determine buy price."):
         freqtrade.execute_buy(pair, stake_amount)
@@ -982,10 +860,6 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order
 
 def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    mocker.patch.multiple(
-        'freqtrade.freqtradebot.FreqtradeBot',
-        get_buy_rate=MagicMock(return_value=0.11),
-    )
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=MagicMock(return_value={
@@ -994,6 +868,7 @@ def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) -
             'last': 0.00001172
         }),
         buy=MagicMock(return_value=limit_buy_order),
+        get_buy_rate=MagicMock(return_value=0.11),
         get_min_pair_stake_amount=MagicMock(return_value=1),
         get_fee=fee,
     )
@@ -1025,7 +900,7 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None
                  return_value=limit_buy_order['amount'])
 
     stoploss = MagicMock(return_value={'id': 13434334})
-    mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss)
+    mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss)
 
     freqtrade = FreqtradeBot(default_conf)
     freqtrade.strategy.order_types['stoploss_on_exchange'] = True
@@ -1057,6 +932,9 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
         buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         sell=MagicMock(return_value={'id': limit_sell_order['id']}),
         get_fee=fee,
+    )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Binance',
         stoploss=stoploss
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -1081,7 +959,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
     trade.stoploss_order_id = 100
 
     hanging_stoploss_order = MagicMock(return_value={'status': 'open'})
-    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', hanging_stoploss_order)
+    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', hanging_stoploss_order)
 
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
     assert trade.stoploss_order_id == 100
@@ -1094,7 +972,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
     trade.stoploss_order_id = 100
 
     canceled_stoploss_order = MagicMock(return_value={'status': 'canceled'})
-    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', canceled_stoploss_order)
+    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', canceled_stoploss_order)
     stoploss.reset_mock()
 
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
@@ -1120,14 +998,14 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
         'average': 2,
         'amount': limit_buy_order['amount'],
     })
-    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hit)
+    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', stoploss_order_hit)
     assert freqtrade.handle_stoploss_on_exchange(trade) is True
     assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog)
     assert trade.stoploss_order_id is None
     assert trade.is_open is False
 
     mocker.patch(
-        'freqtrade.exchange.Exchange.stoploss',
+        'freqtrade.exchange.Binance.stoploss',
         side_effect=ExchangeError()
     )
     trade.is_open = True
@@ -1139,9 +1017,9 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
     # It should try to add stoploss order
     trade.stoploss_order_id = 100
     stoploss.reset_mock()
-    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order',
+    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order',
                  side_effect=InvalidOrderException())
-    mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss)
+    mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss)
     freqtrade.handle_stoploss_on_exchange(trade)
     assert stoploss.call_count == 1
 
@@ -1151,7 +1029,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
     trade.is_open = False
     stoploss.reset_mock()
     mocker.patch('freqtrade.exchange.Exchange.fetch_order')
-    mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss)
+    mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss)
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
     assert stoploss.call_count == 0
 
@@ -1171,6 +1049,9 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog,
         buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         sell=MagicMock(return_value={'id': limit_sell_order['id']}),
         get_fee=fee,
+    )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Binance',
         fetch_stoploss_order=MagicMock(return_value={'status': 'canceled', 'id': 100}),
         stoploss=MagicMock(side_effect=ExchangeError()),
     )
@@ -1205,6 +1086,9 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee,
         buy=MagicMock(return_value=limit_buy_order_open),
         sell=sell_mock,
         get_fee=fee,
+    )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Binance',
         fetch_order=MagicMock(return_value={'status': 'canceled'}),
         stoploss=MagicMock(side_effect=InvalidOrderException()),
     )
@@ -1250,6 +1134,9 @@ def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog,
         sell=sell_mock,
         get_fee=fee,
         fetch_order=MagicMock(return_value={'status': 'canceled'}),
+    )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Binance',
         stoploss=MagicMock(side_effect=InsufficientFundsError()),
     )
     patch_get_signal(freqtrade)
@@ -1287,6 +1174,9 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee,
         buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         sell=MagicMock(return_value={'id': limit_sell_order['id']}),
         get_fee=fee,
+    )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Binance',
         stoploss=stoploss,
         stoploss_adjust=MagicMock(return_value=True),
     )
@@ -1327,7 +1217,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee,
         }
     })
 
-    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hanging)
+    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', stoploss_order_hanging)
 
     # stoploss initially at 5%
     assert freqtrade.handle_trade(trade) is False
@@ -1342,8 +1232,8 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee,
 
     cancel_order_mock = MagicMock()
     stoploss_order_mock = MagicMock(return_value={'id': 13434334})
-    mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', cancel_order_mock)
-    mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss_order_mock)
+    mocker.patch('freqtrade.exchange.Binance.cancel_stoploss_order', cancel_order_mock)
+    mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss_order_mock)
 
     # stoploss should not be updated as the interval is 60 seconds
     assert freqtrade.handle_trade(trade) is False
@@ -1390,6 +1280,9 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c
         buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         sell=MagicMock(return_value={'id': limit_sell_order['id']}),
         get_fee=fee,
+    )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Binance',
         stoploss=stoploss,
         stoploss_adjust=MagicMock(return_value=True),
     )
@@ -1425,9 +1318,10 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c
             'stopPrice': '0.1'
         }
     }
-    mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order',
+    mocker.patch('freqtrade.exchange.Binance.cancel_stoploss_order',
                  side_effect=InvalidOrderException())
-    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hanging)
+    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order',
+                 return_value=stoploss_order_hanging)
     freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging)
     assert log_has_re(r"Could not cancel stoploss order abcd for pair ETH/BTC.*", caplog)
 
@@ -1436,8 +1330,8 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c
 
     # Fail creating stoploss order
     caplog.clear()
-    cancel_mock = mocker.patch("freqtrade.exchange.Exchange.cancel_stoploss_order", MagicMock())
-    mocker.patch("freqtrade.exchange.Exchange.stoploss", side_effect=ExchangeError())
+    cancel_mock = mocker.patch("freqtrade.exchange.Binance.cancel_stoploss_order", MagicMock())
+    mocker.patch("freqtrade.exchange.Binance.stoploss", side_effect=ExchangeError())
     freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging)
     assert cancel_mock.call_count == 1
     assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog)
@@ -1459,6 +1353,9 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee,
         buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         sell=MagicMock(return_value={'id': limit_sell_order['id']}),
         get_fee=fee,
+    )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Binance',
         stoploss=stoploss,
         stoploss_adjust=MagicMock(return_value=True),
     )
@@ -1499,7 +1396,7 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee,
         }
     })
 
-    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hanging)
+    mocker.patch('freqtrade.exchange.Binance.fetch_stoploss_order', stoploss_order_hanging)
 
     assert freqtrade.handle_trade(trade) is False
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
@@ -1513,8 +1410,8 @@ def test_handle_stoploss_on_exchange_custom_stop(mocker, default_conf, fee,
 
     cancel_order_mock = MagicMock()
     stoploss_order_mock = MagicMock(return_value={'id': 13434334})
-    mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', cancel_order_mock)
-    mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss_order_mock)
+    mocker.patch('freqtrade.exchange.Binance.cancel_stoploss_order', cancel_order_mock)
+    mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss_order_mock)
 
     # stoploss should not be updated as the interval is 60 seconds
     assert freqtrade.handle_trade(trade) is False
@@ -1745,6 +1642,7 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No
         open_rate=0.01,
         open_date=arrow.utcnow().datetime,
         amount=11,
+        exchange="binance",
     )
     assert not freqtrade.update_trade_state(trade, None)
     assert log_has_re(r'Orderid for trade .* is empty.', caplog)
@@ -1777,7 +1675,6 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_
     # fetch_order should not be called!!
     mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError))
     patch_exchange(mocker)
-    Trade.session = MagicMock()
     amount = sum(x['amount'] for x in trades_for_order)
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     trade = Trade(
@@ -1803,7 +1700,6 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_
     # fetch_order should not be called!!
     mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError))
     patch_exchange(mocker)
-    Trade.session = MagicMock()
     amount = sum(x['amount'] for x in trades_for_order)
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     trade = Trade(
@@ -1866,7 +1762,6 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde
     mocker.patch('freqtrade.wallets.Wallets.update', wallet_mock)
 
     patch_exchange(mocker)
-    Trade.session = MagicMock()
     amount = limit_sell_order["amount"]
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     wallet_mock.reset_mock()
@@ -2014,7 +1909,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open,
     # if ROI is reached we must sell
     patch_get_signal(freqtrade, value=(False, True))
     assert freqtrade.handle_trade(trade)
-    assert log_has("ETH/BTC - Required profit reached. sell_flag=True, sell_type=SellType.ROI",
+    assert log_has("ETH/BTC - Required profit reached. sell_type=SellType.ROI",
                    caplog)
 
 
@@ -2043,7 +1938,7 @@ def test_handle_trade_use_sell_signal(
 
     patch_get_signal(freqtrade, value=(False, True))
     assert freqtrade.handle_trade(trade)
-    assert log_has("ETH/BTC - Sell signal received. sell_flag=True, sell_type=SellType.SELL_SIGNAL",
+    assert log_has("ETH/BTC - Sell signal received. sell_type=SellType.SELL_SIGNAL",
                    caplog)
 
 
@@ -2108,7 +2003,7 @@ def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_or
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
 
     # Ensure default is to return empty (so not mocked yet)
     freqtrade.check_handle_timedout()
@@ -2159,7 +2054,7 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, op
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
 
     freqtrade.strategy.check_buy_timeout = MagicMock(return_value=False)
     # check it does cancel buy orders over the time limit
@@ -2189,7 +2084,7 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
 
     # check it does cancel buy orders over the time limit
     freqtrade.check_handle_timedout()
@@ -2216,7 +2111,7 @@ def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_ord
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
 
     # check it does cancel buy orders over the time limit
     freqtrade.check_handle_timedout()
@@ -2243,9 +2138,10 @@ def test_check_handle_timedout_sell_usercustom(default_conf, ticker, limit_sell_
 
     open_trade.open_date = arrow.utcnow().shift(hours=-5).datetime
     open_trade.close_date = arrow.utcnow().shift(minutes=-601).datetime
+    open_trade.close_profit_abs = 0.001
     open_trade.is_open = False
 
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
     # Ensure default is false
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 0
@@ -2290,9 +2186,10 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old,
 
     open_trade.open_date = arrow.utcnow().shift(hours=-5).datetime
     open_trade.close_date = arrow.utcnow().shift(minutes=-601).datetime
+    open_trade.close_profit_abs = 0.001
     open_trade.is_open = False
 
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
 
     freqtrade.strategy.check_sell_timeout = MagicMock(return_value=False)
     # check it does cancel sell orders over the time limit
@@ -2323,7 +2220,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old,
     open_trade.close_date = arrow.utcnow().shift(minutes=-601).datetime
     open_trade.is_open = False
 
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
 
     # check it does cancel sell orders over the time limit
     freqtrade.check_handle_timedout()
@@ -2349,13 +2246,13 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
 
     # check it does cancel buy orders over the time limit
     # note this is for a partially-complete buy order
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     assert trades[0].amount == 23.0
@@ -2382,7 +2279,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
 
     open_trade.fee_open = fee()
     open_trade.fee_close = fee()
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
     # cancelling a half-filled order should update the amount to the bought amount
     # and apply fees if necessary.
     freqtrade.check_handle_timedout()
@@ -2390,7 +2287,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
     assert log_has_re(r"Applying fee on amount for Trade.*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -2422,7 +2319,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade,
 
     open_trade.fee_open = fee()
     open_trade.fee_close = fee()
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
     # cancelling a half-filled order should update the amount to the bought amount
     # and apply fees if necessary.
     freqtrade.check_handle_timedout()
@@ -2430,7 +2327,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade,
     assert log_has_re(r"Could not update trade amount: .*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 1
+    assert rpc_mock.call_count == 2
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -2459,7 +2356,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke
     )
     freqtrade = FreqtradeBot(default_conf)
 
-    Trade.session.add(open_trade)
+    Trade.query.session.add(open_trade)
 
     freqtrade.check_handle_timedout()
     assert log_has_re(r"Cannot query order for Trade\(id=1, pair=ETH/BTC, amount=90.99181073, "
@@ -2482,15 +2379,23 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non
     freqtrade = FreqtradeBot(default_conf)
     freqtrade._notify_buy_cancel = MagicMock()
 
-    Trade.session = MagicMock()
     trade = MagicMock()
-    trade.pair = 'LTC/ETH'
+    trade.pair = 'LTC/USDT'
+    trade.open_rate = 200
     limit_buy_order['filled'] = 0.0
     limit_buy_order['status'] = 'open'
     reason = CANCEL_REASON['TIMEOUT']
     assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
     assert cancel_order_mock.call_count == 1
 
+    cancel_order_mock.reset_mock()
+    caplog.clear()
+    limit_buy_order['filled'] = 0.01
+    assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
+    assert cancel_order_mock.call_count == 0
+    assert log_has_re("Order .* for .* not cancelled, as the filled amount.* unsellable.*", caplog)
+
+    caplog.clear()
     cancel_order_mock.reset_mock()
     limit_buy_order['filled'] = 2
     assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
@@ -2516,7 +2421,6 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf,
     nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_buy_cancel')
     freqtrade = FreqtradeBot(default_conf)
 
-    Trade.session = MagicMock()
     reason = CANCEL_REASON['TIMEOUT']
     trade = MagicMock()
     trade.pair = 'LTC/ETH'
@@ -2545,9 +2449,9 @@ def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order,
     freqtrade = FreqtradeBot(default_conf)
     freqtrade._notify_buy_cancel = MagicMock()
 
-    Trade.session = MagicMock()
     trade = MagicMock()
-    trade.pair = 'LTC/ETH'
+    trade.pair = 'LTC/USDT'
+    trade.open_rate = 200
     limit_buy_order['filled'] = 0.0
     limit_buy_order['status'] = 'open'
     reason = CANCEL_REASON['TIMEOUT']
@@ -2568,7 +2472,7 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None:
         'freqtrade.exchange.Exchange',
         cancel_order=cancel_order_mock,
     )
-    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', return_value=0.245441)
+    mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', return_value=0.245441)
 
     freqtrade = FreqtradeBot(default_conf)
 
@@ -2648,22 +2552,24 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N
         fetch_ticker=ticker_sell_up
     )
     # Prevented sell ...
-    freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI)
+    freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'],
+                           sell_reason=SellCheckTuple(sell_type=SellType.ROI))
     assert rpc_mock.call_count == 0
     assert freqtrade.strategy.confirm_trade_exit.call_count == 1
 
     # Repatch with true
     freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=True)
 
-    freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI)
+    freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'],
+                           sell_reason=SellCheckTuple(sell_type=SellType.ROI))
     assert freqtrade.strategy.confirm_trade_exit.call_count == 1
 
     assert rpc_mock.call_count == 1
     last_msg = rpc_mock.call_args_list[-1][0][0]
     assert {
         'trade_id': 1,
-        'type': RPCMessageType.SELL_NOTIFICATION,
-        'exchange': 'Bittrex',
+        'type': RPCMessageType.SELL,
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'gain': 'profit',
         'limit': 1.172e-05,
@@ -2678,6 +2584,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N
         'sell_reason': SellType.ROI.value,
         'open_date': ANY,
         'close_date': ANY,
+        'close_rate': ANY,
     } == last_msg
 
 
@@ -2706,14 +2613,14 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker)
     )
 
     freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'],
-                           sell_reason=SellType.STOP_LOSS)
+                           sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS))
 
     assert rpc_mock.call_count == 2
     last_msg = rpc_mock.call_args_list[-1][0][0]
     assert {
-        'type': RPCMessageType.SELL_NOTIFICATION,
+        'type': RPCMessageType.SELL,
         'trade_id': 1,
-        'exchange': 'Bittrex',
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'gain': 'loss',
         'limit': 1.044e-05,
@@ -2728,6 +2635,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker)
         'sell_reason': SellType.STOP_LOSS.value,
         'open_date': ANY,
         'close_date': ANY,
+        'close_rate': ANY,
     } == last_msg
 
 
@@ -2762,15 +2670,15 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe
 
     trade.stop_loss = 0.00001099 * 0.99
     freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'],
-                           sell_reason=SellType.STOP_LOSS)
+                           sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS))
 
     assert rpc_mock.call_count == 2
     last_msg = rpc_mock.call_args_list[-1][0][0]
 
     assert {
-        'type': RPCMessageType.SELL_NOTIFICATION,
+        'type': RPCMessageType.SELL,
         'trade_id': 1,
-        'exchange': 'Bittrex',
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'gain': 'loss',
         'limit': 1.08801e-05,
@@ -2785,7 +2693,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe
         'sell_reason': SellType.STOP_LOSS.value,
         'open_date': ANY,
         'close_date': ANY,
-
+        'close_rate': ANY,
     } == last_msg
 
 
@@ -2794,7 +2702,7 @@ def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, c
     mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order',
                  side_effect=InvalidOrderException())
     mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=300))
-    sellmock = MagicMock()
+    sellmock = MagicMock(return_value={'id': '12345555'})
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
@@ -2808,14 +2716,13 @@ def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, c
     freqtrade.enter_positions()
 
     trade = Trade.query.first()
-    Trade.session = MagicMock()
     PairLock.session = MagicMock()
 
     freqtrade.config['dry_run'] = False
     trade.stoploss_order_id = "abcd"
 
     freqtrade.execute_sell(trade=trade, limit=1234,
-                           sell_reason=SellType.STOP_LOSS)
+                           sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS))
     assert sellmock.call_count == 1
     assert log_has('Could not cancel stoploss order abcd', caplog)
 
@@ -2865,12 +2772,12 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke
     )
 
     freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'],
-                           sell_reason=SellType.SELL_SIGNAL)
+                           sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS))
 
     trade = Trade.query.first()
     assert trade
     assert cancel_order.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
 
 
 def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, fee,
@@ -2938,7 +2845,10 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f
     assert trade.stoploss_order_id is None
     assert trade.is_open is False
     assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
+    assert rpc_mock.call_args_list[0][0][0]['type'] == RPCMessageType.BUY
+    assert rpc_mock.call_args_list[1][0][0]['type'] == RPCMessageType.BUY_FILL
+    assert rpc_mock.call_args_list[2][0][0]['type'] == RPCMessageType.SELL
 
 
 def test_execute_sell_market_order(default_conf, ticker, fee,
@@ -2967,17 +2877,18 @@ def test_execute_sell_market_order(default_conf, ticker, fee,
     )
     freqtrade.config['order_types']['sell'] = 'market'
 
-    freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI)
+    freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'],
+                           sell_reason=SellCheckTuple(sell_type=SellType.ROI))
 
     assert not trade.is_open
     assert trade.close_profit == 0.0620716
 
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 3
     last_msg = rpc_mock.call_args_list[-1][0][0]
     assert {
-        'type': RPCMessageType.SELL_NOTIFICATION,
+        'type': RPCMessageType.SELL,
         'trade_id': 1,
-        'exchange': 'Bittrex',
+        'exchange': 'Binance',
         'pair': 'ETH/BTC',
         'gain': 'profit',
         'limit': 1.172e-05,
@@ -2992,6 +2903,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee,
         'sell_reason': SellType.ROI.value,
         'open_date': ANY,
         'close_date': ANY,
+        'close_rate': ANY,
 
     } == last_msg
 
@@ -3020,8 +2932,9 @@ def test_execute_sell_insufficient_funds_error(default_conf, ticker, fee,
         fetch_ticker=ticker_sell_up
     )
 
+    sell_reason = SellCheckTuple(sell_type=SellType.ROI)
     assert not freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'],
-                                      sell_reason=SellType.ROI)
+                                      sell_reason=sell_reason)
     assert mock_insuf.call_count == 1
 
 
@@ -3114,7 +3027,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, limit_buy_o
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
     freqtrade.strategy.stop_loss_reached = MagicMock(return_value=SellCheckTuple(
-        sell_flag=False, sell_type=SellType.NONE))
+        sell_type=SellType.NONE))
     freqtrade.enter_positions()
 
     trade = Trade.query.first()
@@ -3263,7 +3176,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo
     )
 
     freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'],
-                           sell_reason=SellType.STOP_LOSS)
+                           sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS))
     trade.close(ticker_sell_down()['bid'])
     assert freqtrade.strategy.is_pair_locked(trade.pair)
 
@@ -3960,7 +3873,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open,
     assert trade.stake_amount == 0.001
     assert trade.is_open
     assert trade.open_date is not None
-    assert trade.exchange == 'bittrex'
+    assert trade.exchange == 'binance'
 
     assert len(Trade.query.all()) == 1
 
@@ -4014,7 +3927,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None:
     default_conf['telegram']['enabled'] = False
 
     freqtrade = FreqtradeBot(default_conf)
-    assert freqtrade.get_buy_rate('ETH/BTC', True) == 0.043935
+    assert freqtrade.exchange.get_buy_rate('ETH/BTC', True) == 0.043935
     assert ticker_mock.call_count == 0
 
 
@@ -4036,7 +3949,7 @@ def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None
     freqtrade = FreqtradeBot(default_conf)
     # orderbook shall be used even if tickers would be lower.
     with pytest.raises(PricingError):
-        freqtrade.get_buy_rate('ETH/BTC', refresh=True)
+        freqtrade.exchange.get_buy_rate('ETH/BTC', refresh=True)
     assert log_has_re(r'Buy Price from orderbook could not be determined.', caplog)
 
 
@@ -4108,97 +4021,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o
     assert log_has('Sell Price at location 1 from orderbook could not be determined.', caplog)
 
 
-@pytest.mark.parametrize('side,ask,bid,expected', [
-    ('bid', 10.0, 11.0, 11.0),
-    ('bid', 10.0, 11.2, 11.2),
-    ('bid', 10.0, 11.0, 11.0),
-    ('bid', 9.8, 11.0, 11.0),
-    ('bid', 0.0001, 0.002, 0.002),
-    ('ask', 10.0, 11.0, 10.0),
-    ('ask', 10.11, 11.2, 10.11),
-    ('ask', 0.001, 0.002, 0.001),
-    ('ask', 0.006, 1.0, 0.006),
-])
-def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, expected) -> None:
-    caplog.set_level(logging.DEBUG)
-
-    default_conf['ask_strategy']['price_side'] = side
-    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'ask': ask, 'bid': bid})
-    pair = "ETH/BTC"
-
-    # Test regular mode
-    ft = get_patched_freqtradebot(mocker, default_conf)
-    rate = ft.get_sell_rate(pair, True)
-    assert not log_has("Using cached sell rate for ETH/BTC.", caplog)
-    assert isinstance(rate, float)
-    assert rate == expected
-    # Use caching
-    rate = ft.get_sell_rate(pair, False)
-    assert rate == expected
-    assert log_has("Using cached sell rate for ETH/BTC.", caplog)
-
-
-@pytest.mark.parametrize('side,expected', [
-    ('bid', 0.043936),  # Value from order_book_l2 fiture - bids side
-    ('ask', 0.043949),  # Value from order_book_l2 fiture - asks side
-])
-def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2):
-    caplog.set_level(logging.DEBUG)
-    # Test orderbook mode
-    default_conf['ask_strategy']['price_side'] = side
-    default_conf['ask_strategy']['use_order_book'] = True
-    default_conf['ask_strategy']['order_book_min'] = 1
-    default_conf['ask_strategy']['order_book_max'] = 2
-    pair = "ETH/BTC"
-    mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2)
-    ft = get_patched_freqtradebot(mocker, default_conf)
-    rate = ft.get_sell_rate(pair, True)
-    assert not log_has("Using cached sell rate for ETH/BTC.", caplog)
-    assert isinstance(rate, float)
-    assert rate == expected
-    rate = ft.get_sell_rate(pair, False)
-    assert rate == expected
-    assert log_has("Using cached sell rate for ETH/BTC.", caplog)
-
-
-def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog):
-    # Test orderbook mode
-    default_conf['ask_strategy']['price_side'] = 'ask'
-    default_conf['ask_strategy']['use_order_book'] = True
-    default_conf['ask_strategy']['order_book_min'] = 1
-    default_conf['ask_strategy']['order_book_max'] = 2
-    pair = "ETH/BTC"
-    # Test What happens if the exchange returns an empty orderbook.
-    mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book',
-                 return_value={'bids': [[]], 'asks': [[]]})
-    ft = get_patched_freqtradebot(mocker, default_conf)
-    with pytest.raises(PricingError):
-        ft.get_sell_rate(pair, True)
-    assert log_has("Sell Price at location from orderbook could not be determined.", caplog)
-
-
-def test_get_sell_rate_exception(default_conf, mocker, caplog):
-    # Ticker on one side can be empty in certain circumstances.
-    default_conf['ask_strategy']['price_side'] = 'ask'
-    pair = "ETH/BTC"
-    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
-                 return_value={'ask': None, 'bid': 0.12})
-    ft = get_patched_freqtradebot(mocker, default_conf)
-    with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."):
-        ft.get_sell_rate(pair, True)
-
-    ft.config['ask_strategy']['price_side'] = 'bid'
-    assert ft.get_sell_rate(pair, True) == 0.12
-    # Reverse sides
-    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
-                 return_value={'ask': 0.13, 'bid': None})
-    with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."):
-        ft.get_sell_rate(pair, True)
-
-    ft.config['ask_strategy']['price_side'] = 'ask'
-    assert ft.get_sell_rate(pair, True) == 0.13
-
-
 def test_startup_state(default_conf, mocker):
     default_conf['pairlist'] = {'method': 'VolumePairList',
                                 'config': {'number_assets': 20}
@@ -4405,9 +4227,9 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog):
         is_open=True,
         amount=20,
         open_rate=0.01,
-        exchange='bittrex',
+        exchange='binance',
     )
-    Trade.session.add(trade)
+    Trade.query.session.add(trade)
 
     freqtrade.reupdate_buy_order_fees(trade)
     assert log_has_re(r"Trying to reupdate buy fees for .*", caplog)
diff --git a/tests/test_integration.py b/tests/test_integration.py
index 8e3bd251a..b12959a03 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -2,9 +2,10 @@ from unittest.mock import MagicMock
 
 import pytest
 
+from freqtrade.enums import SellType
 from freqtrade.persistence import Trade
 from freqtrade.rpc.rpc import RPC
-from freqtrade.strategy.interface import SellCheckTuple, SellType
+from freqtrade.strategy.interface import SellCheckTuple
 from tests.conftest import get_patched_freqtradebot, patch_get_signal
 
 
@@ -51,8 +52,8 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
         side_effect=[stoploss_order_closed, stoploss_order_open, stoploss_order_open])
     # Sell 3rd trade (not called for the first trade)
     should_sell_mock = MagicMock(side_effect=[
-        SellCheckTuple(sell_flag=False, sell_type=SellType.NONE),
-        SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL)]
+        SellCheckTuple(sell_type=SellType.NONE),
+        SellCheckTuple(sell_type=SellType.SELL_SIGNAL)]
     )
     cancel_order_mock = MagicMock()
     mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss)
@@ -63,7 +64,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
         amount_to_precision=lambda s, x, y: y,
         price_to_precision=lambda s, x, y: y,
         fetch_stoploss_order=stoploss_order_mock,
-        cancel_stoploss_order=cancel_order_mock,
+        cancel_stoploss_order_with_result=cancel_order_mock,
     )
 
     mocker.patch.multiple(
@@ -89,7 +90,6 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
     freqtrade.strategy.confirm_trade_entry.reset_mock()
     assert freqtrade.strategy.confirm_trade_exit.call_count == 0
     wallets_mock.reset_mock()
-    Trade.session = MagicMock()
 
     trades = Trade.query.all()
     # Make sure stoploss-order is open and trade is bought (since we mock update_trade_state)
@@ -157,11 +157,11 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc
         _notify_sell=MagicMock(),
     )
     should_sell_mock = MagicMock(side_effect=[
-        SellCheckTuple(sell_flag=False, sell_type=SellType.NONE),
-        SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL),
-        SellCheckTuple(sell_flag=False, sell_type=SellType.NONE),
-        SellCheckTuple(sell_flag=False, sell_type=SellType.NONE),
-        SellCheckTuple(sell_flag=None, sell_type=SellType.NONE)]
+        SellCheckTuple(sell_type=SellType.NONE),
+        SellCheckTuple(sell_type=SellType.SELL_SIGNAL),
+        SellCheckTuple(sell_type=SellType.NONE),
+        SellCheckTuple(sell_type=SellType.NONE),
+        SellCheckTuple(sell_type=SellType.NONE)]
     )
     mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock)
 
@@ -178,8 +178,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc
 
     trades = Trade.query.all()
     assert len(trades) == 4
-    assert freqtrade.wallets.get_trade_stake_amount(
-        'XRP/BTC', freqtrade.get_free_open_trades()) == result1
+    assert freqtrade.wallets.get_trade_stake_amount('XRP/BTC') == result1
 
     rpc._rpc_forcebuy('TKN/BTC', None)
 
@@ -200,8 +199,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc
     # One trade sold
     assert len(trades) == 4
     # stake-amount should now be reduced, since one trade was sold at a loss.
-    assert freqtrade.wallets.get_trade_stake_amount(
-        'XRP/BTC', freqtrade.get_free_open_trades()) < result1
+    assert freqtrade.wallets.get_trade_stake_amount('XRP/BTC') < result1
     # Validate that balance of sold trade is not in dry-run balances anymore.
     bals2 = freqtrade.wallets.get_all_balances()
     assert bals != bals2
diff --git a/tests/test_main.py b/tests/test_main.py
index 70632aeaa..3546a3bab 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -7,10 +7,10 @@ from unittest.mock import MagicMock, PropertyMock
 import pytest
 
 from freqtrade.commands import Arguments
+from freqtrade.enums import State
 from freqtrade.exceptions import FreqtradeException, OperationalException
 from freqtrade.freqtradebot import FreqtradeBot
 from freqtrade.main import main
-from freqtrade.state import State
 from freqtrade.worker import Worker
 from tests.conftest import (log_has, log_has_re, patch_exchange,
                             patched_configuration_load_config_file)
@@ -118,7 +118,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None:
 def test_main_operational_exception1(mocker, default_conf, caplog) -> None:
     patch_exchange(mocker)
     mocker.patch(
-        'freqtrade.commands.list_commands.available_exchanges',
+        'freqtrade.commands.list_commands.validate_exchanges',
         MagicMock(side_effect=ValueError('Oh snap!'))
     )
     patched_configuration_load_config_file(mocker, default_conf)
@@ -132,7 +132,7 @@ def test_main_operational_exception1(mocker, default_conf, caplog) -> None:
     assert log_has('Fatal exception!', caplog)
     assert not log_has_re(r'SIGINT.*', caplog)
     mocker.patch(
-        'freqtrade.commands.list_commands.available_exchanges',
+        'freqtrade.commands.list_commands.validate_exchanges',
         MagicMock(side_effect=KeyboardInterrupt)
     )
     with pytest.raises(SystemExit):
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index d0d29f142..1576aaa5a 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -1,32 +1,36 @@
 # pragma pylint: disable=missing-docstring, C0103
 import logging
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from types import FunctionType
 from unittest.mock import MagicMock
 
 import arrow
 import pytest
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, inspect, text
 
 from freqtrade import constants
 from freqtrade.exceptions import DependencyException, OperationalException
-from freqtrade.persistence import Order, Trade, clean_dry_run_db, init_db
+from freqtrade.persistence import LocalTrade, Order, Trade, clean_dry_run_db, init_db
 from tests.conftest import create_mock_trades, log_has, log_has_re
 
 
 def test_init_create_session(default_conf):
     # Check if init create a session
     init_db(default_conf['db_url'], default_conf['dry_run'])
-    assert hasattr(Trade, 'session')
-    assert 'scoped_session' in type(Trade.session).__name__
+    assert hasattr(Trade, '_session')
+    assert 'scoped_session' in type(Trade._session).__name__
 
 
-def test_init_custom_db_url(default_conf, mocker):
+def test_init_custom_db_url(default_conf, tmpdir):
     # Update path to a value other than default, but still in-memory
-    default_conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'})
-    create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
+    filename = f"{tmpdir}/freqtrade2_test.sqlite"
+    assert not Path(filename).is_file()
+
+    default_conf.update({'db_url': f'sqlite:///{filename}'})
 
     init_db(default_conf['db_url'], default_conf['dry_run'])
-    assert create_engine_mock.call_count == 1
-    assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tmp/freqtrade2_test.sqlite'
+    assert Path(filename).is_file()
 
 
 def test_init_invalid_db_url(default_conf):
@@ -47,19 +51,20 @@ def test_init_prod_db(default_conf, mocker):
     assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.sqlite'
 
 
-def test_init_dryrun_db(default_conf, mocker):
-    default_conf.update({'dry_run': True})
-    default_conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL})
-
-    create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
+def test_init_dryrun_db(default_conf, tmpdir):
+    filename = f"{tmpdir}/freqtrade2_prod.sqlite"
+    assert not Path(filename).is_file()
+    default_conf.update({
+        'dry_run': True,
+        'db_url': f'sqlite:///{filename}'
+    })
 
     init_db(default_conf['db_url'], default_conf['dry_run'])
-    assert create_engine_mock.call_count == 1
-    assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.dryrun.sqlite'
+    assert Path(filename).is_file()
 
 
 @pytest.mark.usefixtures("init_persistence")
-def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog):
+def test_update_with_binance(limit_buy_order, limit_sell_order, fee, caplog):
     """
     On this test we will buy and sell a crypto currency.
 
@@ -97,7 +102,7 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog):
         open_date=arrow.utcnow().datetime,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
     )
     assert trade.open_order_id is None
     assert trade.close_profit is None
@@ -137,7 +142,7 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog):
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_date=arrow.utcnow().datetime,
-        exchange='bittrex',
+        exchange='binance',
     )
 
     trade.open_order_id = 'something'
@@ -172,7 +177,7 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee):
         amount=5,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
     )
 
     trade.open_order_id = 'something'
@@ -200,7 +205,7 @@ def test_trade_close(limit_buy_order, limit_sell_order, fee):
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_date=arrow.Arrow(2020, 2, 1, 15, 5, 1).datetime,
-        exchange='bittrex',
+        exchange='binance',
     )
     assert trade.close_profit is None
     assert trade.close_date is None
@@ -228,7 +233,7 @@ def test_calc_close_trade_price_exception(limit_buy_order, fee):
         amount=5,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
     )
 
     trade.open_order_id = 'something'
@@ -245,7 +250,7 @@ def test_update_open_order(limit_buy_order):
         amount=5,
         fee_open=0.1,
         fee_close=0.1,
-        exchange='bittrex',
+        exchange='binance',
     )
 
     assert trade.open_order_id is None
@@ -269,7 +274,7 @@ def test_update_invalid_order(limit_buy_order):
         open_rate=0.001,
         fee_open=0.1,
         fee_close=0.1,
-        exchange='bittrex',
+        exchange='binance',
     )
     limit_buy_order['type'] = 'invalid'
     with pytest.raises(ValueError, match=r'Unknown order type'):
@@ -285,7 +290,7 @@ def test_calc_open_trade_value(limit_buy_order, fee):
         open_rate=0.00001099,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
     )
     trade.open_order_id = 'open_trade'
     trade.update(limit_buy_order)  # Buy @ 0.00001099
@@ -306,7 +311,7 @@ def test_calc_close_trade_price(limit_buy_order, limit_sell_order, fee):
         open_rate=0.00001099,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
     )
     trade.open_order_id = 'close_trade'
     trade.update(limit_buy_order)  # Buy @ 0.00001099
@@ -331,7 +336,7 @@ def test_calc_profit(limit_buy_order, limit_sell_order, fee):
         open_rate=0.00001099,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
     )
     trade.open_order_id = 'something'
     trade.update(limit_buy_order)  # Buy @ 0.00001099
@@ -365,7 +370,7 @@ def test_calc_profit_ratio(limit_buy_order, limit_sell_order, fee):
         open_rate=0.00001099,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
     )
     trade.open_order_id = 'something'
     trade.update(limit_buy_order)  # Buy @ 0.00001099
@@ -383,6 +388,9 @@ def test_calc_profit_ratio(limit_buy_order, limit_sell_order, fee):
     # Test with a custom fee rate on the close trade
     assert trade.calc_profit_ratio(fee=0.003) == 0.06147824
 
+    trade.open_trade_value = 0.0
+    assert trade.calc_profit_ratio(fee=0.003) == 0.0
+
 
 @pytest.mark.usefixtures("init_persistence")
 def test_clean_dry_run_db(default_conf, fee):
@@ -395,10 +403,10 @@ def test_clean_dry_run_db(default_conf, fee):
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_rate=0.123,
-        exchange='bittrex',
+        exchange='binance',
         open_order_id='dry_run_buy_12345'
     )
-    Trade.session.add(trade)
+    Trade.query.session.add(trade)
 
     trade = Trade(
         pair='ETC/BTC',
@@ -407,10 +415,10 @@ def test_clean_dry_run_db(default_conf, fee):
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_rate=0.123,
-        exchange='bittrex',
+        exchange='binance',
         open_order_id='dry_run_sell_12345'
     )
-    Trade.session.add(trade)
+    Trade.query.session.add(trade)
 
     # Simulate prod entry
     trade = Trade(
@@ -420,10 +428,10 @@ def test_clean_dry_run_db(default_conf, fee):
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_rate=0.123,
-        exchange='bittrex',
+        exchange='binance',
         open_order_id='prod_buy_12345'
     )
-    Trade.session.add(trade)
+    Trade.query.session.add(trade)
 
     # We have 3 entries: 2 dry_run, 1 prod
     assert len(Trade.query.filter(Trade.open_order_id.isnot(None)).all()) == 3
@@ -458,7 +466,7 @@ def test_migrate_old(mocker, default_conf, fee):
                                 );"""
     insert_table_old = """INSERT INTO trades (exchange, pair, is_open, open_order_id, fee,
                           open_rate, stake_amount, amount, open_date)
-                          VALUES ('BITTREX', 'BTC_ETC', 1, '123123', {fee},
+                          VALUES ('binance', 'BTC_ETC', 1, '123123', {fee},
                           0.00258580, {stake}, {amount},
                           '2017-11-28 12:44:24.000000')
                           """.format(fee=fee.return_value,
@@ -467,7 +475,7 @@ def test_migrate_old(mocker, default_conf, fee):
                                      )
     insert_table_old2 = """INSERT INTO trades (exchange, pair, is_open, fee,
                           open_rate, close_rate, stake_amount, amount, open_date)
-                          VALUES ('BITTREX', 'BTC_ETC', 0, {fee},
+                          VALUES ('binance', 'BTC_ETC', 0, {fee},
                           0.00258580, 0.00268580, {stake}, {amount},
                           '2017-11-28 12:44:24.000000')
                           """.format(fee=fee.return_value,
@@ -478,9 +486,10 @@ def test_migrate_old(mocker, default_conf, fee):
     mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
 
     # Create table using the old format
-    engine.execute(create_table_old)
-    engine.execute(insert_table_old)
-    engine.execute(insert_table_old2)
+    with engine.begin() as connection:
+        connection.execute(text(create_table_old))
+        connection.execute(text(insert_table_old))
+        connection.execute(text(insert_table_old2))
     # Run init to test migration
     init_db(default_conf['db_url'], default_conf['dry_run'])
 
@@ -495,7 +504,7 @@ def test_migrate_old(mocker, default_conf, fee):
     assert trade.amount_requested == amount
     assert trade.stake_amount == default_conf.get("stake_amount")
     assert trade.pair == "ETC/BTC"
-    assert trade.exchange == "bittrex"
+    assert trade.exchange == "binance"
     assert trade.max_rate == 0.0
     assert trade.stop_loss == 0.0
     assert trade.initial_stop_loss == 0.0
@@ -571,15 +580,16 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
 
     # Create table using the old format
-    engine.execute(create_table_old)
-    engine.execute("create index ix_trades_is_open on trades(is_open)")
-    engine.execute("create index ix_trades_pair on trades(pair)")
-    engine.execute(insert_table_old)
+    with engine.begin() as connection:
+        connection.execute(text(create_table_old))
+        connection.execute(text("create index ix_trades_is_open on trades(is_open)"))
+        connection.execute(text("create index ix_trades_pair on trades(pair)"))
+        connection.execute(text(insert_table_old))
 
-    # fake previous backup
-    engine.execute("create table trades_bak as select * from trades")
+        # fake previous backup
+        connection.execute(text("create table trades_bak as select * from trades"))
 
-    engine.execute("create table trades_bak1 as select * from trades")
+        connection.execute(text("create table trades_bak1 as select * from trades"))
     # Run init to test migration
     init_db(default_conf['db_url'], default_conf['dry_run'])
 
@@ -619,6 +629,65 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert orders[1].order_id == 'stop_order_id222'
     assert orders[1].ft_order_side == 'stoploss'
 
+    caplog.clear()
+    # Drop latest column
+    with engine.begin() as connection:
+        connection.execute(text("alter table orders rename to orders_bak"))
+    inspector = inspect(engine)
+
+    with engine.begin() as connection:
+        for index in inspector.get_indexes('orders_bak'):
+            connection.execute(text(f"drop index {index['name']}"))
+        # Recreate table
+        connection.execute(text("""
+            CREATE TABLE orders (
+                id INTEGER NOT NULL,
+                ft_trade_id INTEGER,
+                ft_order_side VARCHAR NOT NULL,
+                ft_pair VARCHAR NOT NULL,
+                ft_is_open BOOLEAN NOT NULL,
+                order_id VARCHAR NOT NULL,
+                status VARCHAR,
+                symbol VARCHAR,
+                order_type VARCHAR,
+                side VARCHAR,
+                price FLOAT,
+                amount FLOAT,
+                filled FLOAT,
+                remaining FLOAT,
+                cost FLOAT,
+                order_date DATETIME,
+                order_filled_date DATETIME,
+                order_update_date DATETIME,
+                PRIMARY KEY (id),
+                CONSTRAINT _order_pair_order_id UNIQUE (ft_pair, order_id),
+                FOREIGN KEY(ft_trade_id) REFERENCES trades (id)
+            )
+            """))
+
+        connection.execute(text("""
+        insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status,
+            symbol, order_type, side, price, amount, filled, remaining, cost, order_date,
+            order_filled_date, order_update_date)
+            select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status,
+            symbol, order_type, side, price, amount, filled, remaining, cost, order_date,
+            order_filled_date, order_update_date
+            from orders_bak
+        """))
+
+    # Run init to test migration
+    init_db(default_conf['db_url'], default_conf['dry_run'])
+
+    assert log_has("trying orders_bak1", caplog)
+
+    orders = Order.query.all()
+    assert len(orders) == 2
+    assert orders[0].order_id == 'buy_order'
+    assert orders[0].ft_order_side == 'buy'
+
+    assert orders[1].order_id == 'stop_order_id222'
+    assert orders[1].ft_order_side == 'stoploss'
+
 
 def test_migrate_mid_state(mocker, default_conf, fee, caplog):
     """
@@ -657,8 +726,9 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
     mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
 
     # Create table using the old format
-    engine.execute(create_table_old)
-    engine.execute(insert_table_old)
+    with engine.begin() as connection:
+        connection.execute(text(create_table_old))
+        connection.execute(text(insert_table_old))
 
     # Run init to test migration
     init_db(default_conf['db_url'], default_conf['dry_run'])
@@ -689,7 +759,7 @@ def test_adjust_stop_loss(fee):
         amount=5,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
         open_rate=1,
         max_rate=1,
     )
@@ -741,7 +811,7 @@ def test_adjust_min_max_rates(fee):
         amount=5,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
         open_rate=1,
     )
 
@@ -766,11 +836,16 @@ def test_adjust_min_max_rates(fee):
 
 
 @pytest.mark.usefixtures("init_persistence")
-def test_get_open(fee):
+@pytest.mark.parametrize('use_db', [True, False])
+def test_get_open(fee, use_db):
+    Trade.use_db = use_db
+    Trade.reset_trades()
 
-    create_mock_trades(fee)
+    create_mock_trades(fee, use_db)
     assert len(Trade.get_open_trades()) == 4
 
+    Trade.use_db = True
+
 
 @pytest.mark.usefixtures("init_persistence")
 def test_to_json(default_conf, fee):
@@ -785,7 +860,7 @@ def test_to_json(default_conf, fee):
         fee_close=fee.return_value,
         open_date=arrow.utcnow().shift(hours=-2).datetime,
         open_rate=0.123,
-        exchange='bittrex',
+        exchange='binance',
         open_order_id='dry_run_buy_12345'
     )
     result = trade.to_json()
@@ -794,11 +869,9 @@ def test_to_json(default_conf, fee):
     assert result == {'trade_id': None,
                       'pair': 'ETH/BTC',
                       'is_open': None,
-                      'open_date_hum': '2 hours ago',
                       'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"),
                       'open_timestamp': int(trade.open_date.timestamp() * 1000),
                       'open_order_id': 'dry_run_buy_12345',
-                      'close_date_hum': None,
                       'close_date': None,
                       'close_timestamp': None,
                       'open_rate': 0.123,
@@ -838,7 +911,7 @@ def test_to_json(default_conf, fee):
                       'max_rate': None,
                       'strategy': None,
                       'timeframe': None,
-                      'exchange': 'bittrex',
+                      'exchange': 'binance',
                       }
 
     # Simulate dry_run entries
@@ -853,17 +926,15 @@ def test_to_json(default_conf, fee):
         close_date=arrow.utcnow().shift(hours=-1).datetime,
         open_rate=0.123,
         close_rate=0.125,
-        exchange='bittrex',
+        exchange='binance',
     )
     result = trade.to_json()
     assert isinstance(result, dict)
 
     assert result == {'trade_id': None,
                       'pair': 'XRP/BTC',
-                      'open_date_hum': '2 hours ago',
                       'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"),
                       'open_timestamp': int(trade.open_date.timestamp() * 1000),
-                      'close_date_hum': 'an hour ago',
                       'close_date': trade.close_date.strftime("%Y-%m-%d %H:%M:%S"),
                       'close_timestamp': int(trade.close_date.timestamp() * 1000),
                       'open_rate': 0.123,
@@ -905,7 +976,7 @@ def test_to_json(default_conf, fee):
                       'sell_order_status': None,
                       'strategy': None,
                       'timeframe': None,
-                      'exchange': 'bittrex',
+                      'exchange': 'binance',
                       }
 
 
@@ -918,7 +989,7 @@ def test_stoploss_reinitialization(default_conf, fee):
         open_date=arrow.utcnow().shift(hours=-2).datetime,
         amount=10,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
         open_rate=1,
         max_rate=1,
     )
@@ -928,7 +999,7 @@ def test_stoploss_reinitialization(default_conf, fee):
     assert trade.stop_loss_pct == -0.05
     assert trade.initial_stop_loss == 0.95
     assert trade.initial_stop_loss_pct == -0.05
-    Trade.session.add(trade)
+    Trade.query.session.add(trade)
 
     # Lower stoploss
     Trade.stoploss_reinitialization(0.06)
@@ -977,7 +1048,7 @@ def test_update_fee(fee):
         open_date=arrow.utcnow().shift(hours=-2).datetime,
         amount=10,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
         open_rate=1,
         max_rate=1,
     )
@@ -1016,7 +1087,7 @@ def test_fee_updated(fee):
         open_date=arrow.utcnow().shift(hours=-2).datetime,
         amount=10,
         fee_close=fee.return_value,
-        exchange='bittrex',
+        exchange='binance',
         open_rate=1,
         max_rate=1,
     )
@@ -1039,14 +1110,52 @@ def test_fee_updated(fee):
 
 
 @pytest.mark.usefixtures("init_persistence")
-def test_total_open_trades_stakes(fee):
+@pytest.mark.parametrize('use_db', [True, False])
+def test_total_open_trades_stakes(fee, use_db):
 
+    Trade.use_db = use_db
+    Trade.reset_trades()
     res = Trade.total_open_trades_stakes()
     assert res == 0
-    create_mock_trades(fee)
+    create_mock_trades(fee, use_db)
     res = Trade.total_open_trades_stakes()
     assert res == 0.004
 
+    Trade.use_db = True
+
+
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize('use_db', [True, False])
+def test_get_trades_proxy(fee, use_db):
+    Trade.use_db = use_db
+    Trade.reset_trades()
+    create_mock_trades(fee, use_db)
+    trades = Trade.get_trades_proxy()
+    assert len(trades) == 6
+
+    assert isinstance(trades[0], Trade)
+
+    trades = Trade.get_trades_proxy(is_open=True)
+    assert len(trades) == 4
+    assert trades[0].is_open
+    trades = Trade.get_trades_proxy(is_open=False)
+
+    assert len(trades) == 2
+    assert not trades[0].is_open
+
+    opendate = datetime.now(tz=timezone.utc) - timedelta(minutes=15)
+
+    assert len(Trade.get_trades_proxy(open_date=opendate)) == 3
+
+    Trade.use_db = True
+
+
+def test_get_trades_backtest():
+    Trade.use_db = False
+    with pytest.raises(NotImplementedError, match=r"`Trade.get_trades\(\)` not .*"):
+        Trade.get_trades([])
+    Trade.use_db = True
+
 
 @pytest.mark.usefixtures("init_persistence")
 def test_get_overall_performance(fee):
@@ -1172,3 +1281,39 @@ def test_select_order(fee):
     assert order.ft_order_side == 'stoploss'
     order = trades[4].select_order('sell', False)
     assert order is None
+
+
+def test_Trade_object_idem():
+
+    assert issubclass(Trade, LocalTrade)
+
+    trade = vars(Trade)
+    localtrade = vars(LocalTrade)
+
+    excludes = (
+        'delete',
+        'session',
+        'commit',
+        'query',
+        'open_date',
+        'get_best_pair',
+        'get_overall_performance',
+        'total_open_trades_stakes',
+        'get_sold_trades_without_assigned_fees',
+        'get_open_trades_without_assigned_fees',
+        'get_open_order_trades',
+        'get_trades',
+        )
+
+    # Parent (LocalTrade) should have the same attributes
+    for item in trade:
+        # Exclude private attributes and open_date (as it's not assigned a default)
+        if (not item.startswith('_') and item not in excludes):
+            assert item in localtrade
+
+    # Fails if only a column is added without corresponding parent field
+    for item in localtrade:
+        if (not item.startswith('__')
+                and item not in ('trades', 'trades_open', 'total_profit')
+                and type(getattr(LocalTrade, item)) not in (property, FunctionType)):
+            assert item in trade
diff --git a/tests/test_plotting.py b/tests/test_plotting.py
index 1752f9b94..20f159e3a 100644
--- a/tests/test_plotting.py
+++ b/tests/test_plotting.py
@@ -331,13 +331,13 @@ def test_generate_profit_graph(testdatadir):
 
     trades = trades[trades['pair'].isin(pairs)]
 
-    fig = generate_profit_graph(pairs, data, trades, timeframe="5m")
+    fig = generate_profit_graph(pairs, data, trades, timeframe="5m", stake_currency='BTC')
     assert isinstance(fig, go.Figure)
 
     assert fig.layout.title.text == "Freqtrade Profit plot"
     assert fig.layout.yaxis.title.text == "Price"
-    assert fig.layout.yaxis2.title.text == "Profit"
-    assert fig.layout.yaxis3.title.text == "Profit"
+    assert fig.layout.yaxis2.title.text == "Profit BTC"
+    assert fig.layout.yaxis3.title.text == "Profit BTC"
 
     figure = fig.layout.figure
     assert len(figure.data) == 5
@@ -356,7 +356,8 @@ def test_generate_profit_graph(testdatadir):
 
     with pytest.raises(OperationalException, match=r"No trades found.*"):
         # Pair cannot be empty - so it's an empty dataframe.
-        generate_profit_graph(pairs, data, trades.loc[trades['pair'].isnull()], timeframe="5m")
+        generate_profit_graph(pairs, data, trades.loc[trades['pair'].isnull()], timeframe="5m",
+                              stake_currency='BTC')
 
 
 def test_start_plot_dataframe(mocker):
@@ -459,7 +460,11 @@ def test_plot_profit(default_conf, mocker, testdatadir):
     assert store_mock.call_count == 1
 
     assert profit_mock.call_args_list[0][0][0] == default_conf['pairs']
-    assert store_mock.call_args_list[0][1]['auto_open'] is True
+    assert store_mock.call_args_list[0][1]['auto_open'] is False
+
+    del default_conf['timeframe']
+    with pytest.raises(OperationalException, match=r"Timeframe must be set.*--timeframe.*"):
+        plot_profit(default_conf)
 
 
 @pytest.mark.parametrize("ind1,ind2,plot_conf,exp", [
diff --git a/tests/test_timerange.py b/tests/test_timerange.py
index 5c35535f0..dcdaad09d 100644
--- a/tests/test_timerange.py
+++ b/tests/test_timerange.py
@@ -3,6 +3,7 @@ import arrow
 import pytest
 
 from freqtrade.configuration import TimeRange
+from freqtrade.exceptions import OperationalException
 
 
 def test_parse_timerange_incorrect():
@@ -27,9 +28,13 @@ def test_parse_timerange_incorrect():
     timerange = TimeRange.parse_timerange('-1231006505000')
     assert TimeRange(None, 'date', 0, 1231006505) == timerange
 
-    with pytest.raises(Exception, match=r'Incorrect syntax.*'):
+    with pytest.raises(OperationalException, match=r'Incorrect syntax.*'):
         TimeRange.parse_timerange('-')
 
+    with pytest.raises(OperationalException,
+                       match=r'Start date is after stop date for timerange.*'):
+        TimeRange.parse_timerange('20100523-20100522')
+
 
 def test_subtract_start():
     x = TimeRange('date', 'date', 1274486400, 1438214400)
diff --git a/tests/test_wallets.py b/tests/test_wallets.py
index b7aead0c4..ff303e2ec 100644
--- a/tests/test_wallets.py
+++ b/tests/test_wallets.py
@@ -1,7 +1,12 @@
 # pragma pylint: disable=missing-docstring
+from copy import deepcopy
 from unittest.mock import MagicMock
 
-from tests.conftest import get_patched_freqtradebot
+import pytest
+
+from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
+from freqtrade.exceptions import DependencyException
+from tests.conftest import get_patched_freqtradebot, patch_wallet
 
 
 def test_sync_wallet_at_boot(mocker, default_conf):
@@ -106,3 +111,62 @@ def test_sync_wallet_missing_data(mocker, default_conf):
     assert freqtrade.wallets._wallets['GAS'].used is None
     assert freqtrade.wallets._wallets['GAS'].total == 0.260739
     assert freqtrade.wallets.get_free('GAS') == 0.260739
+
+
+def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None:
+    patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+
+    with pytest.raises(DependencyException, match=r'.*stake amount.*'):
+        freqtrade.wallets.get_trade_stake_amount('ETH/BTC')
+
+
+@pytest.mark.parametrize("balance_ratio,result1,result2", [
+                        (1, 50, 66.66666),
+                        (0.99, 49.5, 66.0),
+                        (0.50, 25, 33.3333),
+])
+def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_ratio, result1,
+                                                 result2, limit_buy_order_open,
+                                                 fee, mocker) -> None:
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        fetch_ticker=ticker,
+        buy=MagicMock(return_value=limit_buy_order_open),
+        get_fee=fee
+    )
+
+    conf = deepcopy(default_conf)
+    conf['stake_amount'] = UNLIMITED_STAKE_AMOUNT
+    conf['dry_run_wallet'] = 100
+    conf['max_open_trades'] = 2
+    conf['tradable_balance_ratio'] = balance_ratio
+
+    freqtrade = get_patched_freqtradebot(mocker, conf)
+
+    # no open trades, order amount should be 'balance / max_open_trades'
+    result = freqtrade.wallets.get_trade_stake_amount('ETH/USDT')
+    assert result == result1
+
+    # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)'
+    freqtrade.execute_buy('ETH/USDT', result)
+
+    result = freqtrade.wallets.get_trade_stake_amount('LTC/USDT')
+    assert result == result1
+
+    # create 2 trades, order amount should be None
+    freqtrade.execute_buy('LTC/BTC', result)
+
+    result = freqtrade.wallets.get_trade_stake_amount('XRP/USDT')
+    assert result == 0
+
+    freqtrade.config['max_open_trades'] = 3
+    freqtrade.config['dry_run_wallet'] = 200
+    freqtrade.wallets.start_cap = 200
+    result = freqtrade.wallets.get_trade_stake_amount('XRP/USDT')
+    assert round(result, 4) == round(result2, 4)
+
+    # set max_open_trades = None, so do not trade
+    freqtrade.config['max_open_trades'] = 0
+    result = freqtrade.wallets.get_trade_stake_amount('NEO/USDT')
+    assert result == 0
diff --git a/tests/test_worker.py b/tests/test_worker.py
index 839f7cdac..c3773d296 100644
--- a/tests/test_worker.py
+++ b/tests/test_worker.py
@@ -3,7 +3,7 @@ import time
 from unittest.mock import MagicMock, PropertyMock
 
 from freqtrade.data.dataprovider import DataProvider
-from freqtrade.state import State
+from freqtrade.enums import State
 from freqtrade.worker import Worker
 from tests.conftest import get_patched_worker, log_has, log_has_re
 
diff --git a/tests/testdata/hyperopt_results_SampleStrategy.pickle b/tests/testdata/hyperopt_results_SampleStrategy.pickle
new file mode 100644
index 000000000..2231de7bf
Binary files /dev/null and b/tests/testdata/hyperopt_results_SampleStrategy.pickle differ
diff --git a/tests/testdata/strategy_SampleStrategy.fthypt b/tests/testdata/strategy_SampleStrategy.fthypt
new file mode 100644
index 000000000..6dc2b9ab1
--- /dev/null
+++ b/tests/testdata/strategy_SampleStrategy.fthypt
@@ -0,0 +1,5 @@
+{"loss":100000,"params_dict":{"mfi-value":"20","fastd-value":"21","adx-value":"26","rsi-value":"23","mfi-enabled":true,"fastd-enabled":false,"adx-enabled":false,"rsi-enabled":true,"trigger":"sar_reversal","sell-mfi-value":"97","sell-fastd-value":"85","sell-adx-value":"55","sell-rsi-value":"76","sell-mfi-enabled":true,"sell-fastd-enabled":false,"sell-adx-enabled":true,"sell-rsi-enabled":true,"sell-trigger":"sell-bb_upper","roi_t1":"34","roi_t2":"28","roi_t3":"32","roi_p1":0.031,"roi_p2":0.033,"roi_p3":0.146,"stoploss":-0.05},"params_details":{"buy":{"mfi-value":"20","fastd-value":"21","adx-value":"26","rsi-value":"23","mfi-enabled":true,"fastd-enabled":false,"adx-enabled":false,"rsi-enabled":true,"trigger":"sar_reversal"},"sell":{"sell-mfi-value":"97","sell-fastd-value":"85","sell-adx-value":"55","sell-rsi-value":"76","sell-mfi-enabled":true,"sell-fastd-enabled":false,"sell-adx-enabled":true,"sell-rsi-enabled":true,"sell-trigger":"sell-bb_upper"},"roi":"{0: 0.21, 32: 0.064, 60: 0.031, 94: 0}","stoploss":{"stoploss":-0.05}},"params_not_optimized":{"buy":{},"sell":{}},"results_metrics":{"trades":[],"locks":[],"best_pair":{"key":"ETH/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},"worst_pair":{"key":"ETH/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},"results_per_pair":[{"key":"ETH/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"LTC/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"ETC/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"XLM/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"TRX/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"ADA/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"TOTAL","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0}],"sell_reason_summary":[],"left_open_trades":[{"key":"TOTAL","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0}],"total_trades":0,"total_volume":0.0,"avg_stake_amount":0,"profit_mean":0,"profit_median":0,"profit_total":0.0,"profit_total_abs":0,"backtest_start":"2018-01-10 07:25:00","backtest_start_ts":1515569100000,"backtest_end":"2018-01-30 04:45:00","backtest_end_ts":1517287500000,"backtest_days":19,"backtest_run_start_ts":1620793107,"backtest_run_end_ts":1620793107,"trades_per_day":0.0,"market_change":0,"pairlist":["ETH/BTC","LTC/BTC","ETC/BTC","XLM/BTC","TRX/BTC","ADA/BTC"],"stake_amount":0.05,"stake_currency":"BTC","stake_currency_decimals":8,"starting_balance":1000,"dry_run_wallet":1000,"final_balance":1000,"max_open_trades":3,"max_open_trades_setting":3,"timeframe":"5m","timerange":"","enable_protections":false,"strategy_name":"SampleStrategy","stoploss":-0.1,"trailing_stop":false,"trailing_stop_positive":null,"trailing_stop_positive_offset":0.0,"trailing_only_offset_is_reached":false,"use_custom_stoploss":false,"minimal_roi":{"60":0.01,"30":0.02,"0":0.04},"use_sell_signal":true,"sell_profit_only":false,"sell_profit_offset":0.0,"ignore_roi_if_buy_signal":false,"backtest_best_day":0,"backtest_worst_day":0,"backtest_best_day_abs":0,"backtest_worst_day_abs":0,"winning_days":0,"draw_days":0,"losing_days":0,"wins":0,"losses":0,"draws":0,"holding_avg":"0:00:00","winner_holding_avg":"0:00:00","loser_holding_avg":"0:00:00","max_drawdown":0.0,"max_drawdown_abs":0.0,"max_drawdown_low":0.0,"max_drawdown_high":0.0,"drawdown_start":"1970-01-01 00:00:00+00:00","drawdown_start_ts":0,"drawdown_end":"1970-01-01 00:00:00+00:00","drawdown_end_ts":0,"csum_min":0,"csum_max":0},"results_explanation":"     0 trades. 0/0/0 Wins/Draws/Losses. Avg profit   0.00%. Median profit   0.00%. Total profit  0.00000000 BTC (   0.00\u03A3%). Avg duration 0:00:00 min.","total_profit":0.0,"current_epoch":1,"is_initial_point":true,"is_best":false}
+{"loss":100000,"params_dict":{"mfi-value":"14","fastd-value":"43","adx-value":"30","rsi-value":"24","mfi-enabled":true,"fastd-enabled":true,"adx-enabled":false,"rsi-enabled":true,"trigger":"sar_reversal","sell-mfi-value":"97","sell-fastd-value":"71","sell-adx-value":"82","sell-rsi-value":"99","sell-mfi-enabled":false,"sell-fastd-enabled":false,"sell-adx-enabled":false,"sell-rsi-enabled":true,"sell-trigger":"sell-bb_upper","roi_t1":"84","roi_t2":"35","roi_t3":"19","roi_p1":0.024,"roi_p2":0.022,"roi_p3":0.061,"stoploss":-0.083},"params_details":{"buy":{"mfi-value":"14","fastd-value":"43","adx-value":"30","rsi-value":"24","mfi-enabled":true,"fastd-enabled":true,"adx-enabled":false,"rsi-enabled":true,"trigger":"sar_reversal"},"sell":{"sell-mfi-value":"97","sell-fastd-value":"71","sell-adx-value":"82","sell-rsi-value":"99","sell-mfi-enabled":false,"sell-fastd-enabled":false,"sell-adx-enabled":false,"sell-rsi-enabled":true,"sell-trigger":"sell-bb_upper"},"roi":"{0: 0.107, 19: 0.046, 54: 0.024, 138: 0}","stoploss":{"stoploss":-0.083}},"params_not_optimized":{"buy":{},"sell":{}},"results_metrics":{"trades":[],"locks":[],"best_pair":{"key":"ETH/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},"worst_pair":{"key":"ETH/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},"results_per_pair":[{"key":"ETH/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"LTC/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"ETC/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"XLM/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"TRX/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"ADA/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"TOTAL","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0}],"sell_reason_summary":[],"left_open_trades":[{"key":"TOTAL","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0,"profit_sum_pct":0.0,"profit_total_abs":0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0}],"total_trades":0,"total_volume":0.0,"avg_stake_amount":0,"profit_mean":0,"profit_median":0,"profit_total":0.0,"profit_total_abs":0,"backtest_start":"2018-01-10 07:25:00","backtest_start_ts":1515569100000,"backtest_end":"2018-01-30 04:45:00","backtest_end_ts":1517287500000,"backtest_days":19,"backtest_run_start_ts":1620793107,"backtest_run_end_ts":1620793108,"trades_per_day":0.0,"market_change":0,"pairlist":["ETH/BTC","LTC/BTC","ETC/BTC","XLM/BTC","TRX/BTC","ADA/BTC"],"stake_amount":0.05,"stake_currency":"BTC","stake_currency_decimals":8,"starting_balance":1000,"dry_run_wallet":1000,"final_balance":1000,"max_open_trades":3,"max_open_trades_setting":3,"timeframe":"5m","timerange":"","enable_protections":false,"strategy_name":"SampleStrategy","stoploss":-0.1,"trailing_stop":false,"trailing_stop_positive":null,"trailing_stop_positive_offset":0.0,"trailing_only_offset_is_reached":false,"use_custom_stoploss":false,"minimal_roi":{"60":0.01,"30":0.02,"0":0.04},"use_sell_signal":true,"sell_profit_only":false,"sell_profit_offset":0.0,"ignore_roi_if_buy_signal":false,"backtest_best_day":0,"backtest_worst_day":0,"backtest_best_day_abs":0,"backtest_worst_day_abs":0,"winning_days":0,"draw_days":0,"losing_days":0,"wins":0,"losses":0,"draws":0,"holding_avg":"0:00:00","winner_holding_avg":"0:00:00","loser_holding_avg":"0:00:00","max_drawdown":0.0,"max_drawdown_abs":0.0,"max_drawdown_low":0.0,"max_drawdown_high":0.0,"drawdown_start":"1970-01-01 00:00:00+00:00","drawdown_start_ts":0,"drawdown_end":"1970-01-01 00:00:00+00:00","drawdown_end_ts":0,"csum_min":0,"csum_max":0},"results_explanation":"     0 trades. 0/0/0 Wins/Draws/Losses. Avg profit   0.00%. Median profit   0.00%. Total profit  0.00000000 BTC (   0.00\u03A3%). Avg duration 0:00:00 min.","total_profit":0.0,"current_epoch":2,"is_initial_point":true,"is_best":false}
+{"loss":2.183447401951895,"params_dict":{"mfi-value":"14","fastd-value":"15","adx-value":"40","rsi-value":"36","mfi-enabled":false,"fastd-enabled":true,"adx-enabled":false,"rsi-enabled":false,"trigger":"sar_reversal","sell-mfi-value":"92","sell-fastd-value":"84","sell-adx-value":"61","sell-rsi-value":"61","sell-mfi-enabled":true,"sell-fastd-enabled":true,"sell-adx-enabled":true,"sell-rsi-enabled":true,"sell-trigger":"sell-bb_upper","roi_t1":"68","roi_t2":"41","roi_t3":"21","roi_p1":0.015,"roi_p2":0.064,"roi_p3":0.126,"stoploss":-0.024},"params_details":{"buy":{"mfi-value":"14","fastd-value":"15","adx-value":"40","rsi-value":"36","mfi-enabled":false,"fastd-enabled":true,"adx-enabled":false,"rsi-enabled":false,"trigger":"sar_reversal"},"sell":{"sell-mfi-value":"92","sell-fastd-value":"84","sell-adx-value":"61","sell-rsi-value":"61","sell-mfi-enabled":true,"sell-fastd-enabled":true,"sell-adx-enabled":true,"sell-rsi-enabled":true,"sell-trigger":"sell-bb_upper"},"roi":"{0: 0.20500000000000002, 21: 0.079, 62: 0.015, 130: 0}","stoploss":{"stoploss":-0.024}},"params_not_optimized":{"buy":{},"sell":{}},"results_metrics":{"trades":[{"pair":"LTC/BTC","stake_amount":0.05,"amount":2.94115571,"open_date":"2018-01-11 11:40:00+00:00","close_date":"2018-01-11 19:40:00+00:00","open_rate":0.01700012,"close_rate":0.017119538805820372,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":480,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.01659211712,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.01659211712,"stop_loss_ratio":-0.024,"min_rate":0.01689809,"max_rate":0.0171462,"is_open":false,"open_timestamp":1515670800000.0,"close_timestamp":1515699600000.0},{"pair":"ETH/BTC","stake_amount":0.05,"amount":0.57407318,"open_date":"2018-01-12 11:05:00+00:00","close_date":"2018-01-12 12:30:00+00:00","open_rate":0.08709691,"close_rate":0.08901977203712995,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":85,"profit_ratio":0.01494768,"profit_abs":0.00075,"sell_reason":"roi","initial_stop_loss_abs":0.08500658416,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.08500658416,"stop_loss_ratio":-0.024,"min_rate":0.08702974000000001,"max_rate":0.08929248000000001,"is_open":false,"open_timestamp":1515755100000.0,"close_timestamp":1515760200000.0},{"pair":"LTC/BTC","stake_amount":0.05,"amount":2.93166236,"open_date":"2018-01-12 03:30:00+00:00","close_date":"2018-01-12 13:05:00+00:00","open_rate":0.01705517,"close_rate":0.01717497550928249,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":575,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.016645845920000003,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.016645845920000003,"stop_loss_ratio":-0.024,"min_rate":0.0169841,"max_rate":0.01719135,"is_open":false,"open_timestamp":1515727800000.0,"close_timestamp":1515762300000.0},{"pair":"LTC/BTC","stake_amount":0.05,"amount":2.96876855,"open_date":"2018-01-13 03:50:00+00:00","close_date":"2018-01-13 06:05:00+00:00","open_rate":0.016842,"close_rate":0.016960308078273957,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":135,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.016437792,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.016437792,"stop_loss_ratio":-0.024,"min_rate":0.016836999999999998,"max_rate":0.017,"is_open":false,"open_timestamp":1515815400000.0,"close_timestamp":1515823500000.0},{"pair":"ETH/BTC","stake_amount":0.05,"amount":0.53163205,"open_date":"2018-01-13 13:25:00+00:00","close_date":"2018-01-13 15:35:00+00:00","open_rate":0.09405001,"close_rate":0.09471067238835926,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":130,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.09179280976,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.09179280976,"stop_loss_ratio":-0.024,"min_rate":0.09369894000000001,"max_rate":0.09479997,"is_open":false,"open_timestamp":1515849900000.0,"close_timestamp":1515857700000.0},{"pair":"ETC/BTC","stake_amount":0.05,"amount":19.23816853,"open_date":"2018-01-13 15:30:00+00:00","close_date":"2018-01-13 16:20:00+00:00","open_rate":0.0025989999999999997,"close_rate":0.0028232990466633217,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":50,"profit_ratio":0.07872446,"profit_abs":0.00395,"sell_reason":"roi","initial_stop_loss_abs":0.002536624,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.002536624,"stop_loss_ratio":-0.024,"min_rate":0.00259525,"max_rate":0.0028288700000000003,"is_open":false,"open_timestamp":1515857400000.0,"close_timestamp":1515860400000.0},{"pair":"TRX/BTC","stake_amount":0.05,"amount":492.80504632,"open_date":"2018-01-14 21:35:00+00:00","close_date":"2018-01-14 23:15:00+00:00","open_rate":0.00010146000000000001,"close_rate":0.00010369995985950828,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":100,"profit_ratio":0.01494768,"profit_abs":0.00075,"sell_reason":"roi","initial_stop_loss_abs":9.902496e-05,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":9.902496e-05,"stop_loss_ratio":-0.024,"min_rate":0.0001012,"max_rate":0.00010414,"is_open":false,"open_timestamp":1515965700000.0,"close_timestamp":1515971700000.0},{"pair":"LTC/BTC","stake_amount":0.05,"amount":2.92398174,"open_date":"2018-01-15 12:45:00+00:00","close_date":"2018-01-15 21:05:00+00:00","open_rate":0.01709997,"close_rate":0.01722009021073758,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":500,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.016689570719999998,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.016689570719999998,"stop_loss_ratio":-0.024,"min_rate":0.01694,"max_rate":0.01725,"is_open":false,"open_timestamp":1516020300000.0,"close_timestamp":1516050300000.0},{"pair":"XLM/BTC","stake_amount":0.05,"amount":1111.60515785,"open_date":"2018-01-15 19:50:00+00:00","close_date":"2018-01-15 23:45:00+00:00","open_rate":4.4980000000000006e-05,"close_rate":4.390048e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":235,"profit_ratio":-0.03080817,"profit_abs":-0.0015458,"sell_reason":"stop_loss","initial_stop_loss_abs":4.390048e-05,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":4.390048e-05,"stop_loss_ratio":-0.024,"min_rate":4.409e-05,"max_rate":4.502e-05,"is_open":false,"open_timestamp":1516045800000.0,"close_timestamp":1516059900000.0},{"pair":"TRX/BTC","stake_amount":0.05,"amount":519.80455349,"open_date":"2018-01-21 03:55:00+00:00","close_date":"2018-01-21 04:05:00+00:00","open_rate":9.619e-05,"close_rate":9.388144e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":10,"profit_ratio":-0.03080817,"profit_abs":-0.0015458,"sell_reason":"stop_loss","initial_stop_loss_abs":9.388144e-05,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":9.388144e-05,"stop_loss_ratio":-0.024,"min_rate":9.568e-05,"max_rate":9.673e-05,"is_open":false,"open_timestamp":1516506900000.0,"close_timestamp":1516507500000.0},{"pair":"LTC/BTC","stake_amount":0.05,"amount":3.029754,"open_date":"2018-01-20 22:15:00+00:00","close_date":"2018-01-21 07:45:00+00:00","open_rate":0.01650299,"close_rate":0.016106918239999997,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":570,"profit_ratio":-0.03080817,"profit_abs":-0.0015458,"sell_reason":"stop_loss","initial_stop_loss_abs":0.016106918239999997,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.016106918239999997,"stop_loss_ratio":-0.024,"min_rate":0.0162468,"max_rate":0.01663179,"is_open":false,"open_timestamp":1516486500000.0,"close_timestamp":1516520700000.0},{"pair":"ETC/BTC","stake_amount":0.05,"amount":18.75461832,"open_date":"2018-01-21 13:00:00+00:00","close_date":"2018-01-21 16:25:00+00:00","open_rate":0.00266601,"close_rate":0.00260202576,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":205,"profit_ratio":-0.03080817,"profit_abs":-0.0015458,"sell_reason":"stop_loss","initial_stop_loss_abs":0.00260202576,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.00260202576,"stop_loss_ratio":-0.024,"min_rate":0.0026290800000000002,"max_rate":0.00269384,"is_open":false,"open_timestamp":1516539600000.0,"close_timestamp":1516551900000.0},{"pair":"TRX/BTC","stake_amount":0.05,"amount":552.18111541,"open_date":"2018-01-22 02:10:00+00:00","close_date":"2018-01-22 04:20:00+00:00","open_rate":9.055e-05,"close_rate":9.118607626693427e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":130,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":8.83768e-05,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":8.83768e-05,"stop_loss_ratio":-0.024,"min_rate":9.013e-05,"max_rate":9.197e-05,"is_open":false,"open_timestamp":1516587000000.0,"close_timestamp":1516594800000.0},{"pair":"LTC/BTC","stake_amount":0.05,"amount":2.99733237,"open_date":"2018-01-22 03:20:00+00:00","close_date":"2018-01-22 13:50:00+00:00","open_rate":0.0166815,"close_rate":0.016281143999999997,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":630,"profit_ratio":-0.03080817,"profit_abs":-0.0015458,"sell_reason":"stop_loss","initial_stop_loss_abs":0.016281143999999997,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.016281143999999997,"stop_loss_ratio":-0.024,"min_rate":0.01641443,"max_rate":0.016800000000000002,"is_open":false,"open_timestamp":1516591200000.0,"close_timestamp":1516629000000.0},{"pair":"TRX/BTC","stake_amount":0.05,"amount":503.52467271,"open_date":"2018-01-23 08:55:00+00:00","close_date":"2018-01-23 09:40:00+00:00","open_rate":9.93e-05,"close_rate":9.69168e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":45,"profit_ratio":-0.03080817,"profit_abs":-0.0015458,"sell_reason":"stop_loss","initial_stop_loss_abs":9.69168e-05,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":9.69168e-05,"stop_loss_ratio":-0.024,"min_rate":9.754e-05,"max_rate":0.00010025,"is_open":false,"open_timestamp":1516697700000.0,"close_timestamp":1516700400000.0},{"pair":"ETH/BTC","stake_amount":0.05,"amount":0.55148073,"open_date":"2018-01-24 02:10:00+00:00","close_date":"2018-01-24 04:40:00+00:00","open_rate":0.090665,"close_rate":0.09130188409433015,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":150,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.08848903999999999,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.08848903999999999,"stop_loss_ratio":-0.024,"min_rate":0.090665,"max_rate":0.09146000000000001,"is_open":false,"open_timestamp":1516759800000.0,"close_timestamp":1516768800000.0},{"pair":"ETC/BTC","stake_amount":0.05,"amount":19.10584639,"open_date":"2018-01-24 19:20:00+00:00","close_date":"2018-01-24 21:35:00+00:00","open_rate":0.002617,"close_rate":0.0026353833416959357,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":135,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.002554192,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.002554192,"stop_loss_ratio":-0.024,"min_rate":0.002617,"max_rate":0.00264999,"is_open":false,"open_timestamp":1516821600000.0,"close_timestamp":1516829700000.0},{"pair":"ETC/BTC","stake_amount":0.05,"amount":19.34602691,"open_date":"2018-01-25 14:35:00+00:00","close_date":"2018-01-25 16:35:00+00:00","open_rate":0.00258451,"close_rate":0.002641568926241846,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":120,"profit_ratio":0.01494768,"profit_abs":0.00075,"sell_reason":"roi","initial_stop_loss_abs":0.00252248176,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.00252248176,"stop_loss_ratio":-0.024,"min_rate":0.00258451,"max_rate":0.00264579,"is_open":false,"open_timestamp":1516890900000.0,"close_timestamp":1516898100000.0},{"pair":"LTC/BTC","stake_amount":0.05,"amount":3.11910295,"open_date":"2018-01-24 23:05:00+00:00","close_date":"2018-01-25 16:55:00+00:00","open_rate":0.01603025,"close_rate":0.016142855870546913,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":1070,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.015645523999999997,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.015645523999999997,"stop_loss_ratio":-0.024,"min_rate":0.015798760000000002,"max_rate":0.01617,"is_open":false,"open_timestamp":1516835100000.0,"close_timestamp":1516899300000.0},{"pair":"TRX/BTC","stake_amount":0.05,"amount":553.70985604,"open_date":"2018-01-26 19:30:00+00:00","close_date":"2018-01-26 23:30:00+00:00","open_rate":9.03e-05,"close_rate":9.093432012042147e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":240,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":8.813279999999999e-05,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":8.813279999999999e-05,"stop_loss_ratio":-0.024,"min_rate":8.961e-05,"max_rate":9.1e-05,"is_open":false,"open_timestamp":1516995000000.0,"close_timestamp":1517009400000.0},{"pair":"ETC/BTC","stake_amount":0.05,"amount":19.22929005,"open_date":"2018-01-26 21:15:00+00:00","close_date":"2018-01-28 03:50:00+00:00","open_rate":0.0026002,"close_rate":0.0026184653286502758,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":1835,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.0025377952,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.0025377952,"stop_loss_ratio":-0.024,"min_rate":0.00254702,"max_rate":0.00262797,"is_open":false,"open_timestamp":1517001300000.0,"close_timestamp":1517111400000.0},{"pair":"LTC/BTC","stake_amount":0.05,"amount":3.15677093,"open_date":"2018-01-27 22:05:00+00:00","close_date":"2018-01-28 10:45:00+00:00","open_rate":0.01583897,"close_rate":0.015950232207727046,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":760,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.01545883472,"initial_stop_loss_ratio":-0.024,"stop_loss_abs":0.01545883472,"stop_loss_ratio":-0.024,"min_rate":0.015700000000000002,"max_rate":0.01596521,"is_open":false,"open_timestamp":1517090700000.0,"close_timestamp":1517136300000.0}],"locks":[],"best_pair":{"key":"ETC/BTC","trades":5,"profit_mean":0.012572794000000002,"profit_mean_pct":1.2572794000000003,"profit_sum":0.06286397,"profit_sum_pct":6.29,"profit_total_abs":0.0031542000000000002,"profit_total":3.1542000000000002e-06,"profit_total_pct":0.0,"duration_avg":"7:49:00","wins":2,"draws":2,"losses":1},"worst_pair":{"key":"LTC/BTC","trades":8,"profit_mean":-0.0077020425,"profit_mean_pct":-0.77020425,"profit_sum":-0.06161634,"profit_sum_pct":-6.16,"profit_total_abs":-0.0030916,"profit_total":-3.0915999999999998e-06,"profit_total_pct":-0.0,"duration_avg":"9:50:00","wins":0,"draws":6,"losses":2},"results_per_pair":[{"key":"ETC/BTC","trades":5,"profit_mean":0.012572794000000002,"profit_mean_pct":1.2572794000000003,"profit_sum":0.06286397,"profit_sum_pct":6.29,"profit_total_abs":0.0031542000000000002,"profit_total":3.1542000000000002e-06,"profit_total_pct":0.0,"duration_avg":"7:49:00","wins":2,"draws":2,"losses":1},{"key":"ETH/BTC","trades":3,"profit_mean":0.00498256,"profit_mean_pct":0.498256,"profit_sum":0.01494768,"profit_sum_pct":1.49,"profit_total_abs":0.00075,"profit_total":7.5e-07,"profit_total_pct":0.0,"duration_avg":"2:02:00","wins":1,"draws":2,"losses":0},{"key":"ADA/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0.0,"profit_sum_pct":0.0,"profit_total_abs":0.0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"XLM/BTC","trades":1,"profit_mean":-0.03080817,"profit_mean_pct":-3.080817,"profit_sum":-0.03080817,"profit_sum_pct":-3.08,"profit_total_abs":-0.0015458,"profit_total":-1.5457999999999999e-06,"profit_total_pct":-0.0,"duration_avg":"3:55:00","wins":0,"draws":0,"losses":1},{"key":"TRX/BTC","trades":5,"profit_mean":-0.009333732,"profit_mean_pct":-0.9333732000000001,"profit_sum":-0.04666866,"profit_sum_pct":-4.67,"profit_total_abs":-0.0023416,"profit_total":-2.3416e-06,"profit_total_pct":-0.0,"duration_avg":"1:45:00","wins":1,"draws":2,"losses":2},{"key":"LTC/BTC","trades":8,"profit_mean":-0.0077020425,"profit_mean_pct":-0.77020425,"profit_sum":-0.06161634,"profit_sum_pct":-6.16,"profit_total_abs":-0.0030916,"profit_total":-3.0915999999999998e-06,"profit_total_pct":-0.0,"duration_avg":"9:50:00","wins":0,"draws":6,"losses":2},{"key":"TOTAL","trades":22,"profit_mean":-0.0027855236363636365,"profit_mean_pct":-0.27855236363636365,"profit_sum":-0.06128152,"profit_sum_pct":-6.13,"profit_total_abs":-0.0030748,"profit_total":-3.0747999999999998e-06,"profit_total_pct":-0.0,"duration_avg":"6:12:00","wins":4,"draws":12,"losses":6}],"sell_reason_summary":[{"sell_reason":"roi","trades":16,"wins":4,"draws":12,"losses":0,"profit_mean":0.00772296875,"profit_mean_pct":0.77,"profit_sum":0.1235675,"profit_sum_pct":12.36,"profit_total_abs":0.006200000000000001,"profit_total":0.041189166666666666,"profit_total_pct":4.12},{"sell_reason":"stop_loss","trades":6,"wins":0,"draws":0,"losses":6,"profit_mean":-0.03080817,"profit_mean_pct":-3.08,"profit_sum":-0.18484902,"profit_sum_pct":-18.48,"profit_total_abs":-0.0092748,"profit_total":-0.06161634,"profit_total_pct":-6.16}],"left_open_trades":[{"key":"TOTAL","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0.0,"profit_sum_pct":0.0,"profit_total_abs":0.0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0}],"total_trades":22,"total_volume":1.1000000000000003,"avg_stake_amount":0.05000000000000002,"profit_mean":-0.0027855236363636365,"profit_median":0.0,"profit_total":-3.0747999999999998e-06,"profit_total_abs":-0.0030748,"backtest_start":"2018-01-10 07:25:00","backtest_start_ts":1515569100000,"backtest_end":"2018-01-30 04:45:00","backtest_end_ts":1517287500000,"backtest_days":19,"backtest_run_start_ts":1620793107,"backtest_run_end_ts":1620793108,"trades_per_day":1.16,"market_change":0,"pairlist":["ETH/BTC","LTC/BTC","ETC/BTC","XLM/BTC","TRX/BTC","ADA/BTC"],"stake_amount":0.05,"stake_currency":"BTC","stake_currency_decimals":8,"starting_balance":1000,"dry_run_wallet":1000,"final_balance":999.9969252,"max_open_trades":3,"max_open_trades_setting":3,"timeframe":"5m","timerange":"","enable_protections":false,"strategy_name":"SampleStrategy","stoploss":-0.1,"trailing_stop":false,"trailing_stop_positive":null,"trailing_stop_positive_offset":0.0,"trailing_only_offset_is_reached":false,"use_custom_stoploss":false,"minimal_roi":{"60":0.01,"30":0.02,"0":0.04},"use_sell_signal":true,"sell_profit_only":false,"sell_profit_offset":0.0,"ignore_roi_if_buy_signal":false,"backtest_best_day":0.07872446,"backtest_worst_day":-0.09242451,"backtest_best_day_abs":0.00395,"backtest_worst_day_abs":-0.0046374,"winning_days":4,"draw_days":10,"losing_days":4,"wins":4,"losses":6,"draws":12,"holding_avg":"6:12:00","winner_holding_avg":"1:29:00","loser_holding_avg":"4:42:00","max_drawdown":0.18484901999999998,"max_drawdown_abs":0.0092748,"drawdown_start":"2018-01-14 23:15:00","drawdown_start_ts":1515971700000.0,"drawdown_end":"2018-01-23 09:40:00","drawdown_end_ts":1516700400000.0,"max_drawdown_low":-0.0038247999999999997,"max_drawdown_high":0.00545,"csum_min":999.9961752,"csum_max":1000.00545},"results_explanation":"    22 trades. 4/12/6 Wins/Draws/Losses. Avg profit  -0.28%. Median profit   0.00%. Total profit -0.00307480 BTC (  -0.00\u03A3%). Avg duration 6:12:00 min.","total_profit":-3.0747999999999998e-06,"current_epoch":3,"is_initial_point":true,"is_best":true}
+{"loss":-4.9544427978437175,"params_dict":{"mfi-value":"23","fastd-value":"40","adx-value":"50","rsi-value":"27","mfi-enabled":false,"fastd-enabled":true,"adx-enabled":true,"rsi-enabled":true,"trigger":"bb_lower","sell-mfi-value":"87","sell-fastd-value":"60","sell-adx-value":"81","sell-rsi-value":"69","sell-mfi-enabled":true,"sell-fastd-enabled":true,"sell-adx-enabled":false,"sell-rsi-enabled":false,"sell-trigger":"sell-sar_reversal","roi_t1":"105","roi_t2":"43","roi_t3":"12","roi_p1":0.03,"roi_p2":0.036,"roi_p3":0.103,"stoploss":-0.081},"params_details":{"buy":{"mfi-value":"23","fastd-value":"40","adx-value":"50","rsi-value":"27","mfi-enabled":false,"fastd-enabled":true,"adx-enabled":true,"rsi-enabled":true,"trigger":"bb_lower"},"sell":{"sell-mfi-value":"87","sell-fastd-value":"60","sell-adx-value":"81","sell-rsi-value":"69","sell-mfi-enabled":true,"sell-fastd-enabled":true,"sell-adx-enabled":false,"sell-rsi-enabled":false,"sell-trigger":"sell-sar_reversal"},"roi":"{0: 0.16899999999999998, 12: 0.066, 55: 0.03, 160: 0}","stoploss":{"stoploss":-0.081}},"params_not_optimized":{"buy":{},"sell":{}},"results_metrics":{"trades":[{"pair":"XLM/BTC","stake_amount":0.05,"amount":1086.95652174,"open_date":"2018-01-13 13:30:00+00:00","close_date":"2018-01-13 16:30:00+00:00","open_rate":4.6e-05,"close_rate":4.632313095835424e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":180,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":4.2274e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":4.2274e-05,"stop_loss_ratio":-0.081,"min_rate":4.4980000000000006e-05,"max_rate":4.673e-05,"is_open":false,"open_timestamp":1515850200000.0,"close_timestamp":1515861000000.0},{"pair":"ADA/BTC","stake_amount":0.05,"amount":851.35365231,"open_date":"2018-01-15 14:50:00+00:00","close_date":"2018-01-15 16:15:00+00:00","open_rate":5.873000000000001e-05,"close_rate":6.0910642247867544e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":85,"profit_ratio":0.02989537,"profit_abs":0.0015,"sell_reason":"roi","initial_stop_loss_abs":5.397287000000001e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":5.397287000000001e-05,"stop_loss_ratio":-0.081,"min_rate":5.873000000000001e-05,"max_rate":6.120000000000001e-05,"is_open":false,"open_timestamp":1516027800000.0,"close_timestamp":1516032900000.0},{"pair":"ADA/BTC","stake_amount":0.05,"amount":896.86098655,"open_date":"2018-01-16 00:35:00+00:00","close_date":"2018-01-16 03:15:00+00:00","open_rate":5.575000000000001e-05,"close_rate":5.6960000000000004e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":160,"profit_ratio":0.01457705,"profit_abs":0.0007314,"sell_reason":"roi","initial_stop_loss_abs":5.123425000000001e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":5.123425000000001e-05,"stop_loss_ratio":-0.081,"min_rate":5.575000000000001e-05,"max_rate":5.730000000000001e-05,"is_open":false,"open_timestamp":1516062900000.0,"close_timestamp":1516072500000.0},{"pair":"TRX/BTC","stake_amount":0.05,"amount":747.160789,"open_date":"2018-01-16 22:30:00+00:00","close_date":"2018-01-16 22:45:00+00:00","open_rate":6.692e-05,"close_rate":7.182231811339689e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":15,"profit_ratio":0.06576981,"profit_abs":0.0033,"sell_reason":"roi","initial_stop_loss_abs":6.149948000000001e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":6.149948000000001e-05,"stop_loss_ratio":-0.081,"min_rate":6.692e-05,"max_rate":7.566e-05,"is_open":false,"open_timestamp":1516141800000.0,"close_timestamp":1516142700000.0},{"pair":"TRX/BTC","stake_amount":0.05,"amount":720.5649229,"open_date":"2018-01-17 15:15:00+00:00","close_date":"2018-01-17 16:40:00+00:00","open_rate":6.939000000000001e-05,"close_rate":7.19664475664827e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":85,"profit_ratio":0.02989537,"profit_abs":0.0015,"sell_reason":"roi","initial_stop_loss_abs":6.376941000000001e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":6.376941000000001e-05,"stop_loss_ratio":-0.081,"min_rate":6.758e-05,"max_rate":7.244e-05,"is_open":false,"open_timestamp":1516202100000.0,"close_timestamp":1516207200000.0},{"pair":"XLM/BTC","stake_amount":0.05,"amount":1144.42664225,"open_date":"2018-01-18 22:20:00+00:00","close_date":"2018-01-19 00:35:00+00:00","open_rate":4.3690000000000004e-05,"close_rate":4.531220772704466e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":135,"profit_ratio":0.02989537,"profit_abs":0.0015,"sell_reason":"roi","initial_stop_loss_abs":4.015111e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":4.015111e-05,"stop_loss_ratio":-0.081,"min_rate":4.3690000000000004e-05,"max_rate":4.779e-05,"is_open":false,"open_timestamp":1516314000000.0,"close_timestamp":1516322100000.0},{"pair":"ADA/BTC","stake_amount":0.05,"amount":876.57784011,"open_date":"2018-01-18 22:25:00+00:00","close_date":"2018-01-19 01:05:00+00:00","open_rate":5.704e-05,"close_rate":5.792e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":160,"profit_ratio":0.00834457,"profit_abs":0.00041869,"sell_reason":"roi","initial_stop_loss_abs":5.2419760000000006e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":5.2419760000000006e-05,"stop_loss_ratio":-0.081,"min_rate":5.704e-05,"max_rate":5.8670000000000006e-05,"is_open":false,"open_timestamp":1516314300000.0,"close_timestamp":1516323900000.0},{"pair":"TRX/BTC","stake_amount":0.05,"amount":525.59655209,"open_date":"2018-01-20 05:05:00+00:00","close_date":"2018-01-20 06:25:00+00:00","open_rate":9.513e-05,"close_rate":9.86621726041144e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":80,"profit_ratio":0.02989537,"profit_abs":0.0015,"sell_reason":"roi","initial_stop_loss_abs":8.742447000000001e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":8.742447000000001e-05,"stop_loss_ratio":-0.081,"min_rate":9.513e-05,"max_rate":9.95e-05,"is_open":false,"open_timestamp":1516424700000.0,"close_timestamp":1516429500000.0},{"pair":"ADA/BTC","stake_amount":0.05,"amount":920.64076597,"open_date":"2018-01-26 07:40:00+00:00","close_date":"2018-01-26 10:20:00+00:00","open_rate":5.431000000000001e-05,"close_rate":5.474000000000001e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":160,"profit_ratio":0.0008867,"profit_abs":4.449e-05,"sell_reason":"roi","initial_stop_loss_abs":4.991089000000001e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":4.991089000000001e-05,"stop_loss_ratio":-0.081,"min_rate":5.3670000000000006e-05,"max_rate":5.5e-05,"is_open":false,"open_timestamp":1516952400000.0,"close_timestamp":1516962000000.0},{"pair":"XLM/BTC","stake_amount":0.05,"amount":944.28706327,"open_date":"2018-01-28 04:35:00+00:00","close_date":"2018-01-30 04:45:00+00:00","open_rate":5.2950000000000006e-05,"close_rate":4.995000000000001e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":2890,"profit_ratio":-0.06323759,"profit_abs":-0.00317295,"sell_reason":"force_sell","initial_stop_loss_abs":4.866105000000001e-05,"initial_stop_loss_ratio":-0.081,"stop_loss_abs":4.866105000000001e-05,"stop_loss_ratio":-0.081,"min_rate":4.980000000000001e-05,"max_rate":5.3280000000000005e-05,"is_open":true,"open_timestamp":1517114100000.0,"close_timestamp":1517287500000.0}],"locks":[],"best_pair":{"key":"TRX/BTC","trades":3,"profit_mean":0.04185351666666667,"profit_mean_pct":4.185351666666667,"profit_sum":0.12556055,"profit_sum_pct":12.56,"profit_total_abs":0.0063,"profit_total":6.3e-06,"profit_total_pct":0.0,"duration_avg":"1:00:00","wins":3,"draws":0,"losses":0},"worst_pair":{"key":"XLM/BTC","trades":3,"profit_mean":-0.01111407333333333,"profit_mean_pct":-1.111407333333333,"profit_sum":-0.03334221999999999,"profit_sum_pct":-3.33,"profit_total_abs":-0.0016729499999999999,"profit_total":-1.6729499999999998e-06,"profit_total_pct":-0.0,"duration_avg":"17:48:00","wins":1,"draws":1,"losses":1},"results_per_pair":[{"key":"TRX/BTC","trades":3,"profit_mean":0.04185351666666667,"profit_mean_pct":4.185351666666667,"profit_sum":0.12556055,"profit_sum_pct":12.56,"profit_total_abs":0.0063,"profit_total":6.3e-06,"profit_total_pct":0.0,"duration_avg":"1:00:00","wins":3,"draws":0,"losses":0},{"key":"ADA/BTC","trades":4,"profit_mean":0.0134259225,"profit_mean_pct":1.34259225,"profit_sum":0.05370369,"profit_sum_pct":5.37,"profit_total_abs":0.00269458,"profit_total":2.69458e-06,"profit_total_pct":0.0,"duration_avg":"2:21:00","wins":4,"draws":0,"losses":0},{"key":"ETH/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0.0,"profit_sum_pct":0.0,"profit_total_abs":0.0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"LTC/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0.0,"profit_sum_pct":0.0,"profit_total_abs":0.0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"ETC/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0.0,"profit_sum_pct":0.0,"profit_total_abs":0.0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"XLM/BTC","trades":3,"profit_mean":-0.01111407333333333,"profit_mean_pct":-1.111407333333333,"profit_sum":-0.03334221999999999,"profit_sum_pct":-3.33,"profit_total_abs":-0.0016729499999999999,"profit_total":-1.6729499999999998e-06,"profit_total_pct":-0.0,"duration_avg":"17:48:00","wins":1,"draws":1,"losses":1},{"key":"TOTAL","trades":10,"profit_mean":0.014592201999999999,"profit_mean_pct":1.4592201999999999,"profit_sum":0.14592201999999999,"profit_sum_pct":14.59,"profit_total_abs":0.00732163,"profit_total":7.32163e-06,"profit_total_pct":0.0,"duration_avg":"6:35:00","wins":8,"draws":1,"losses":1}],"sell_reason_summary":[{"sell_reason":"roi","trades":9,"wins":8,"draws":1,"losses":0,"profit_mean":0.023239956666666665,"profit_mean_pct":2.32,"profit_sum":0.20915961,"profit_sum_pct":20.92,"profit_total_abs":0.01049458,"profit_total":0.06971987,"profit_total_pct":6.97},{"sell_reason":"force_sell","trades":1,"wins":0,"draws":0,"losses":1,"profit_mean":-0.06323759,"profit_mean_pct":-6.32,"profit_sum":-0.06323759,"profit_sum_pct":-6.32,"profit_total_abs":-0.00317295,"profit_total":-0.021079196666666664,"profit_total_pct":-2.11}],"left_open_trades":[{"key":"XLM/BTC","trades":1,"profit_mean":-0.06323759,"profit_mean_pct":-6.323759,"profit_sum":-0.06323759,"profit_sum_pct":-6.32,"profit_total_abs":-0.00317295,"profit_total":-3.17295e-06,"profit_total_pct":-0.0,"duration_avg":"2 days, 0:10:00","wins":0,"draws":0,"losses":1},{"key":"TOTAL","trades":1,"profit_mean":-0.06323759,"profit_mean_pct":-6.323759,"profit_sum":-0.06323759,"profit_sum_pct":-6.32,"profit_total_abs":-0.00317295,"profit_total":-3.17295e-06,"profit_total_pct":-0.0,"duration_avg":"2 days, 0:10:00","wins":0,"draws":0,"losses":1}],"total_trades":10,"total_volume":0.5,"avg_stake_amount":0.05,"profit_mean":0.014592201999999999,"profit_median":0.02223621,"profit_total":7.32163e-06,"profit_total_abs":0.00732163,"backtest_start":"2018-01-10 07:25:00","backtest_start_ts":1515569100000,"backtest_end":"2018-01-30 04:45:00","backtest_end_ts":1517287500000,"backtest_days":19,"backtest_run_start_ts":1620793107,"backtest_run_end_ts":1620793108,"trades_per_day":0.53,"market_change":0,"pairlist":["ETH/BTC","LTC/BTC","ETC/BTC","XLM/BTC","TRX/BTC","ADA/BTC"],"stake_amount":0.05,"stake_currency":"BTC","stake_currency_decimals":8,"starting_balance":1000,"dry_run_wallet":1000,"final_balance":1000.00732163,"max_open_trades":3,"max_open_trades_setting":3,"timeframe":"5m","timerange":"","enable_protections":false,"strategy_name":"SampleStrategy","stoploss":-0.1,"trailing_stop":false,"trailing_stop_positive":null,"trailing_stop_positive_offset":0.0,"trailing_only_offset_is_reached":false,"use_custom_stoploss":false,"minimal_roi":{"60":0.01,"30":0.02,"0":0.04},"use_sell_signal":true,"sell_profit_only":false,"sell_profit_offset":0.0,"ignore_roi_if_buy_signal":false,"backtest_best_day":0.08034685999999999,"backtest_worst_day":-0.06323759,"backtest_best_day_abs":0.0040314,"backtest_worst_day_abs":-0.00317295,"winning_days":6,"draw_days":11,"losing_days":1,"wins":8,"losses":1,"draws":1,"holding_avg":"6:35:00","winner_holding_avg":"1:50:00","loser_holding_avg":"2 days, 0:10:00","max_drawdown":0.06323759000000001,"max_drawdown_abs":0.00317295,"drawdown_start":"2018-01-26 10:20:00","drawdown_start_ts":1516962000000.0,"drawdown_end":"2018-01-30 04:45:00","drawdown_end_ts":1517287500000.0,"max_drawdown_low":0.007321629999999998,"max_drawdown_high":0.010494579999999998,"csum_min":1000.0,"csum_max":1000.01049458},"results_explanation":"    10 trades. 8/1/1 Wins/Draws/Losses. Avg profit   1.46%. Median profit   2.22%. Total profit  0.00732163 BTC (   0.00\u03A3%). Avg duration 6:35:00 min.","total_profit":7.32163e-06,"current_epoch":4,"is_initial_point":true,"is_best":true}
+{"loss":0.16709185414267655,"params_dict":{"mfi-value":"10","fastd-value":"45","adx-value":"28","rsi-value":"37","mfi-enabled":false,"fastd-enabled":false,"adx-enabled":true,"rsi-enabled":true,"trigger":"macd_cross_signal","sell-mfi-value":"85","sell-fastd-value":"56","sell-adx-value":"98","sell-rsi-value":"89","sell-mfi-enabled":true,"sell-fastd-enabled":false,"sell-adx-enabled":true,"sell-rsi-enabled":false,"sell-trigger":"sell-sar_reversal","roi_t1":"85","roi_t2":"11","roi_t3":"24","roi_p1":0.04,"roi_p2":0.043,"roi_p3":0.053,"stoploss":-0.057},"params_details":{"buy":{"mfi-value":"10","fastd-value":"45","adx-value":"28","rsi-value":"37","mfi-enabled":false,"fastd-enabled":false,"adx-enabled":true,"rsi-enabled":true,"trigger":"macd_cross_signal"},"sell":{"sell-mfi-value":"85","sell-fastd-value":"56","sell-adx-value":"98","sell-rsi-value":"89","sell-mfi-enabled":true,"sell-fastd-enabled":false,"sell-adx-enabled":true,"sell-rsi-enabled":false,"sell-trigger":"sell-sar_reversal"},"roi":"{0: 0.13599999999999998, 24: 0.08299999999999999, 35: 0.04, 120: 0}","stoploss":{"stoploss":-0.057}},"params_not_optimized":{"buy":{},"sell":{}},"results_metrics":{"trades":[{"pair":"ETH/BTC","stake_amount":0.05,"amount":0.56173464,"open_date":"2018-01-10 19:15:00+00:00","close_date":"2018-01-10 21:15:00+00:00","open_rate":0.08901,"close_rate":0.09112999000000001,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":120,"profit_ratio":0.01667571,"profit_abs":0.0008367,"sell_reason":"roi","initial_stop_loss_abs":0.08393643,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":0.08393643,"stop_loss_ratio":-0.057,"min_rate":0.08894498,"max_rate":0.09116998,"is_open":false,"open_timestamp":1515611700000.0,"close_timestamp":1515618900000.0},{"pair":"ADA/BTC","stake_amount":0.05,"amount":794.65988557,"open_date":"2018-01-13 11:30:00+00:00","close_date":"2018-01-13 15:10:00+00:00","open_rate":6.292e-05,"close_rate":5.9333559999999994e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":220,"profit_ratio":-0.06357798,"profit_abs":-0.00319003,"sell_reason":"stop_loss","initial_stop_loss_abs":5.9333559999999994e-05,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":5.9333559999999994e-05,"stop_loss_ratio":-0.057,"min_rate":5.9900000000000006e-05,"max_rate":6.353e-05,"is_open":false,"open_timestamp":1515843000000.0,"close_timestamp":1515856200000.0},{"pair":"XLM/BTC","stake_amount":0.05,"amount":1086.95652174,"open_date":"2018-01-13 14:35:00+00:00","close_date":"2018-01-13 21:40:00+00:00","open_rate":4.6e-05,"close_rate":4.632313095835424e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":425,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":4.3378e-05,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":4.3378e-05,"stop_loss_ratio":-0.057,"min_rate":4.4980000000000006e-05,"max_rate":4.6540000000000005e-05,"is_open":false,"open_timestamp":1515854100000.0,"close_timestamp":1515879600000.0},{"pair":"ETH/BTC","stake_amount":0.05,"amount":0.53757603,"open_date":"2018-01-15 13:15:00+00:00","close_date":"2018-01-15 15:15:00+00:00","open_rate":0.0930101,"close_rate":0.09366345745107878,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":120,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.0877085243,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":0.0877085243,"stop_loss_ratio":-0.057,"min_rate":0.09188489999999999,"max_rate":0.09380000000000001,"is_open":false,"open_timestamp":1516022100000.0,"close_timestamp":1516029300000.0},{"pair":"ETC/BTC","stake_amount":0.05,"amount":17.07469496,"open_date":"2018-01-15 14:35:00+00:00","close_date":"2018-01-15 16:35:00+00:00","open_rate":0.00292831,"close_rate":0.00297503,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":120,"profit_ratio":0.00886772,"profit_abs":0.00044494,"sell_reason":"roi","initial_stop_loss_abs":0.0027613963299999997,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":0.0027613963299999997,"stop_loss_ratio":-0.057,"min_rate":0.00292831,"max_rate":0.00301259,"is_open":false,"open_timestamp":1516026900000.0,"close_timestamp":1516034100000.0},{"pair":"TRX/BTC","stake_amount":0.05,"amount":702.44450688,"open_date":"2018-01-17 04:25:00+00:00","close_date":"2018-01-17 05:00:00+00:00","open_rate":7.118e-05,"close_rate":7.453721023582538e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":35,"profit_ratio":0.03986049,"profit_abs":0.002,"sell_reason":"roi","initial_stop_loss_abs":6.712274e-05,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":6.712274e-05,"stop_loss_ratio":-0.057,"min_rate":7.118e-05,"max_rate":7.658000000000002e-05,"is_open":false,"open_timestamp":1516163100000.0,"close_timestamp":1516165200000.0},{"pair":"ETC/BTC","stake_amount":0.05,"amount":18.86756854,"open_date":"2018-01-20 06:05:00+00:00","close_date":"2018-01-20 08:05:00+00:00","open_rate":0.00265005,"close_rate":0.00266995,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":120,"profit_ratio":0.00048133,"profit_abs":2.415e-05,"sell_reason":"roi","initial_stop_loss_abs":0.00249899715,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":0.00249899715,"stop_loss_ratio":-0.057,"min_rate":0.00265005,"max_rate":0.00271,"is_open":false,"open_timestamp":1516428300000.0,"close_timestamp":1516435500000.0},{"pair":"ADA/BTC","stake_amount":0.05,"amount":966.18357488,"open_date":"2018-01-22 03:25:00+00:00","close_date":"2018-01-22 07:05:00+00:00","open_rate":5.1750000000000004e-05,"close_rate":5.211352232814853e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":220,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":4.8800250000000004e-05,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":4.8800250000000004e-05,"stop_loss_ratio":-0.057,"min_rate":5.1750000000000004e-05,"max_rate":5.2170000000000004e-05,"is_open":false,"open_timestamp":1516591500000.0,"close_timestamp":1516604700000.0},{"pair":"ETC/BTC","stake_amount":0.05,"amount":18.95303438,"open_date":"2018-01-23 13:10:00+00:00","close_date":"2018-01-23 16:00:00+00:00","open_rate":0.0026381,"close_rate":0.002656631560461616,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":170,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":0.0024877283,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":0.0024877283,"stop_loss_ratio":-0.057,"min_rate":0.0026100000000000003,"max_rate":0.00266,"is_open":false,"open_timestamp":1516713000000.0,"close_timestamp":1516723200000.0},{"pair":"ADA/BTC","stake_amount":0.05,"amount":912.40875912,"open_date":"2018-01-26 06:30:00+00:00","close_date":"2018-01-26 10:45:00+00:00","open_rate":5.480000000000001e-05,"close_rate":5.518494731560462e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":255,"profit_ratio":-0.0,"profit_abs":-0.0,"sell_reason":"roi","initial_stop_loss_abs":5.1676400000000006e-05,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":5.1676400000000006e-05,"stop_loss_ratio":-0.057,"min_rate":5.3670000000000006e-05,"max_rate":5.523e-05,"is_open":false,"open_timestamp":1516948200000.0,"close_timestamp":1516963500000.0},{"pair":"ADA/BTC","stake_amount":0.05,"amount":909.58704748,"open_date":"2018-01-27 02:10:00+00:00","close_date":"2018-01-27 05:40:00+00:00","open_rate":5.4970000000000004e-05,"close_rate":5.535614149523332e-05,"fee_open":0.0035,"fee_close":0.0035,"trade_duration":210,"profit_ratio":0.0,"profit_abs":0.0,"sell_reason":"roi","initial_stop_loss_abs":5.183671e-05,"initial_stop_loss_ratio":-0.057,"stop_loss_abs":5.183671e-05,"stop_loss_ratio":-0.057,"min_rate":5.472000000000001e-05,"max_rate":5.556e-05,"is_open":false,"open_timestamp":1517019000000.0,"close_timestamp":1517031600000.0}],"locks":[],"best_pair":{"key":"TRX/BTC","trades":1,"profit_mean":0.03986049,"profit_mean_pct":3.986049,"profit_sum":0.03986049,"profit_sum_pct":3.99,"profit_total_abs":0.002,"profit_total":2e-06,"profit_total_pct":0.0,"duration_avg":"0:35:00","wins":1,"draws":0,"losses":0},"worst_pair":{"key":"ADA/BTC","trades":4,"profit_mean":-0.015894495,"profit_mean_pct":-1.5894495000000002,"profit_sum":-0.06357798,"profit_sum_pct":-6.36,"profit_total_abs":-0.00319003,"profit_total":-3.19003e-06,"profit_total_pct":-0.0,"duration_avg":"3:46:00","wins":0,"draws":3,"losses":1},"results_per_pair":[{"key":"TRX/BTC","trades":1,"profit_mean":0.03986049,"profit_mean_pct":3.986049,"profit_sum":0.03986049,"profit_sum_pct":3.99,"profit_total_abs":0.002,"profit_total":2e-06,"profit_total_pct":0.0,"duration_avg":"0:35:00","wins":1,"draws":0,"losses":0},{"key":"ETH/BTC","trades":2,"profit_mean":0.008337855,"profit_mean_pct":0.8337855,"profit_sum":0.01667571,"profit_sum_pct":1.67,"profit_total_abs":0.0008367,"profit_total":8.367e-07,"profit_total_pct":0.0,"duration_avg":"2:00:00","wins":1,"draws":1,"losses":0},{"key":"ETC/BTC","trades":3,"profit_mean":0.0031163500000000004,"profit_mean_pct":0.31163500000000005,"profit_sum":0.009349050000000001,"profit_sum_pct":0.93,"profit_total_abs":0.00046909,"profit_total":4.6909000000000003e-07,"profit_total_pct":0.0,"duration_avg":"2:17:00","wins":2,"draws":1,"losses":0},{"key":"LTC/BTC","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0.0,"profit_sum_pct":0.0,"profit_total_abs":0.0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0},{"key":"XLM/BTC","trades":1,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0.0,"profit_sum_pct":0.0,"profit_total_abs":0.0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"7:05:00","wins":0,"draws":1,"losses":0},{"key":"ADA/BTC","trades":4,"profit_mean":-0.015894495,"profit_mean_pct":-1.5894495000000002,"profit_sum":-0.06357798,"profit_sum_pct":-6.36,"profit_total_abs":-0.00319003,"profit_total":-3.19003e-06,"profit_total_pct":-0.0,"duration_avg":"3:46:00","wins":0,"draws":3,"losses":1},{"key":"TOTAL","trades":11,"profit_mean":0.00020975181818181756,"profit_mean_pct":0.020975181818181757,"profit_sum":0.002307269999999993,"profit_sum_pct":0.23,"profit_total_abs":0.00011576000000000034,"profit_total":1.1576000000000034e-07,"profit_total_pct":0.0,"duration_avg":"3:03:00","wins":4,"draws":6,"losses":1}],"sell_reason_summary":[{"sell_reason":"roi","trades":10,"wins":4,"draws":6,"losses":0,"profit_mean":0.0065885250000000005,"profit_mean_pct":0.66,"profit_sum":0.06588525,"profit_sum_pct":6.59,"profit_total_abs":0.0033057900000000003,"profit_total":0.021961750000000002,"profit_total_pct":2.2},{"sell_reason":"stop_loss","trades":1,"wins":0,"draws":0,"losses":1,"profit_mean":-0.06357798,"profit_mean_pct":-6.36,"profit_sum":-0.06357798,"profit_sum_pct":-6.36,"profit_total_abs":-0.00319003,"profit_total":-0.021192660000000002,"profit_total_pct":-2.12}],"left_open_trades":[{"key":"TOTAL","trades":0,"profit_mean":0.0,"profit_mean_pct":0.0,"profit_sum":0.0,"profit_sum_pct":0.0,"profit_total_abs":0.0,"profit_total":0.0,"profit_total_pct":0.0,"duration_avg":"0:00","wins":0,"draws":0,"losses":0}],"total_trades":11,"total_volume":0.55,"avg_stake_amount":0.05,"profit_mean":0.00020975181818181756,"profit_median":0.0,"profit_total":1.1576000000000034e-07,"profit_total_abs":0.00011576000000000034,"backtest_start":"2018-01-10 07:25:00","backtest_start_ts":1515569100000,"backtest_end":"2018-01-30 04:45:00","backtest_end_ts":1517287500000,"backtest_days":19,"backtest_run_start_ts":1620793107,"backtest_run_end_ts":1620793108,"trades_per_day":0.58,"market_change":0,"pairlist":["ETH/BTC","LTC/BTC","ETC/BTC","XLM/BTC","TRX/BTC","ADA/BTC"],"stake_amount":0.05,"stake_currency":"BTC","stake_currency_decimals":8,"starting_balance":1000,"dry_run_wallet":1000,"final_balance":1000.00011576,"max_open_trades":3,"max_open_trades_setting":3,"timeframe":"5m","timerange":"","enable_protections":false,"strategy_name":"SampleStrategy","stoploss":-0.1,"trailing_stop":false,"trailing_stop_positive":null,"trailing_stop_positive_offset":0.0,"trailing_only_offset_is_reached":false,"use_custom_stoploss":false,"minimal_roi":{"60":0.01,"30":0.02,"0":0.04},"use_sell_signal":true,"sell_profit_only":false,"sell_profit_offset":0.0,"ignore_roi_if_buy_signal":false,"backtest_best_day":0.03986049,"backtest_worst_day":-0.06357798,"backtest_best_day_abs":0.002,"backtest_worst_day_abs":-0.00319003,"winning_days":4,"draw_days":13,"losing_days":1,"wins":4,"losses":1,"draws":6,"holding_avg":"3:03:00","winner_holding_avg":"1:39:00","loser_holding_avg":"3:40:00","max_drawdown":0.06357798,"max_drawdown_abs":0.00319003,"drawdown_start":"2018-01-10 21:15:00","drawdown_start_ts":1515618900000.0,"drawdown_end":"2018-01-13 15:10:00","drawdown_end_ts":1515856200000.0,"max_drawdown_low":-0.00235333,"max_drawdown_high":0.0008367,"csum_min":999.99764667,"csum_max":1000.0008367},"results_explanation":"    11 trades. 4/6/1 Wins/Draws/Losses. Avg profit   0.02%. Median profit   0.00%. Total profit  0.00011576 BTC (   0.00\u03A3%). Avg duration 3:03:00 min.","total_profit":1.1576000000000034e-07,"current_epoch":5,"is_initial_point":true,"is_best":false}