diff --git a/.dependabot/config.yml b/.dependabot/config.yml deleted file mode 100644 index 66b91e99f..000000000 --- a/.dependabot/config.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: 1 - -update_configs: - - package_manager: "python" - directory: "/" - update_schedule: "weekly" - allowed_updates: - - match: - update_type: "all" - target_branch: "develop" - - - package_manager: "docker" - directory: "/" - update_schedule: "daily" - allowed_updates: - - match: - update_type: "all" diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..b333dc19d --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,18 @@ +FROM freqtradeorg/freqtrade:develop + +# Install dependencies +COPY requirements-dev.txt /freqtrade/ +RUN apt-get update \ + && apt-get -y install git sudo vim \ + && apt-get clean \ + && pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \ + && useradd -u 1000 -U -m ftuser \ + && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \ + && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \ + && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \ + && chown ftuser: -R /home/ftuser/ + +USER ftuser + +# Empty the ENTRYPOINT to allow all commands +ENTRYPOINT [] diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..1882e3bdf --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,44 @@ +{ + "name": "freqtrade Develop", + + "dockerComposeFile": [ + "docker-compose.yml" + ], + + "service": "ft_vscode", + + "workspaceFolder": "/freqtrade/", + + "settings": { + "terminal.integrated.shell.linux": "/bin/bash", + "editor.insertSpaces": true, + "files.trimTrailingWhitespace": true, + "[markdown]": { + "files.trimTrailingWhitespace": false, + }, + "python.pythonPath": "/usr/local/bin/python", + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance", + "davidanson.vscode-markdownlint", + "ms-azuretools.vscode-docker", + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment the next line if you want start specific services in your Docker Compose config. + // "runServices": [], + + // Uncomment the next line if you want to keep your containers running after VS Code shuts down. + // "shutdownAction": "none", + + // Uncomment the next line to run commands after the container is created - for example installing curl. + // "postCreateCommand": "sudo apt-get update && apt-get install -y git", + + // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "ftuser" +} diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..20ec247d1 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,24 @@ +--- +version: '3' +services: + ft_vscode: + build: + context: .. + dockerfile: ".devcontainer/Dockerfile" + volumes: + # Allow git usage within container + - "${HOME}/.ssh:/home/ftuser/.ssh:ro" + - "${HOME}/.gitconfig:/home/ftuser/.gitconfig:ro" + - ..:/freqtrade:cached + # Persist bash-history + - freqtrade-vscode-server:/home/ftuser/.vscode-server + - freqtrade-bashhistory:/home/ftuser/commandhistory + # Expose API port + ports: + - "127.0.0.1:8080:8080" + command: /bin/sh -c "while sleep 1000; do :; done" + + +volumes: + freqtrade-vscode-server: + freqtrade-bashhistory: diff --git a/.dockerignore b/.dockerignore index 223b3b110..09f4c9f0c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,4 @@ CONTRIBUTING.md MANIFEST.in README.md freqtrade.service +user_data diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..44ff606b4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: +- package-ecosystem: docker + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 +- package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 10 + target-branch: develop diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 239576c61..36a9fc374 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,48 +4,132 @@ on: push: branches: - master + - stable - develop - - github_actions_tests tags: - release: - types: [published] + release: + types: [published] pull_request: schedule: - cron: '0 5 * * 4' jobs: - build: + build_linux: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-18.04, macos-latest ] - python-version: [3.7, 3.8] + os: [ ubuntu-18.04, ubuntu-20.04 ] + python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Cache_dependencies - uses: actions/cache@v1 + uses: actions/cache@v2 id: cache with: path: ~/dependencies/ key: ${{ runner.os }}-dependencies - name: pip cache (linux) - uses: actions/cache@preview + uses: actions/cache@v2 if: startsWith(matrix.os, 'ubuntu') with: path: ~/.cache/pip key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip + - name: TA binary *nix + if: steps.cache.outputs.cache-hit != 'true' + run: | + cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. + + - name: Installation - *nix + run: | + python -m pip install --upgrade pip + export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH + export TA_LIBRARY_PATH=${HOME}/dependencies/lib + export TA_INCLUDE_PATH=${HOME}/dependencies/include + pip install -r requirements-dev.txt + pip install -e . + + - name: Tests + run: | + pytest --random-order --cov=freqtrade --cov-config=.coveragerc + + - name: Coveralls + if: (startsWith(matrix.os, 'ubuntu-20') && matrix.python-version == '3.8') + env: + # Coveralls token. Not used as secret due to github not providing secrets to forked repositories + COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu + run: | + # Allow failure for coveralls + coveralls -v || true + + - name: Backtesting + run: | + cp config.json.example config.json + freqtrade create-userdir --userdir user_data + freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy + + - name: Hyperopt + run: | + cp config.json.example config.json + freqtrade create-userdir --userdir user_data + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + + - name: Flake8 + run: | + flake8 + + - name: Sort imports (isort) + run: | + isort --check . + + - name: Mypy + run: | + mypy freqtrade scripts + + - name: Slack Notification + uses: homoluctus/slatify@v1.8.0 + if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) + with: + type: ${{ job.status }} + job_name: '*Freqtrade CI ${{ matrix.os }}*' + mention: 'here' + mention_if: 'failure' + channel: '#notifications' + url: ${{ secrets.SLACK_WEBHOOK }} + + build_macos: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ macos-latest ] + python-version: [3.7, 3.8] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache_dependencies + uses: actions/cache@v2 + id: cache + with: + path: ~/dependencies/ + key: ${{ runner.os }}-dependencies + - name: pip cache (macOS) - uses: actions/cache@preview + uses: actions/cache@v2 if: startsWith(matrix.os, 'macOS') with: path: ~/Library/Caches/pip @@ -70,7 +154,7 @@ jobs: pytest --random-order --cov=freqtrade --cov-config=.coveragerc - name: Coveralls - if: (startsWith(matrix.os, 'ubuntu') && matrix.python-version == '3.8') + if: (startsWith(matrix.os, 'ubuntu-20') && matrix.python-version == '3.8') env: # Coveralls token. Not used as secret due to github not providing secrets to forked repositories COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu @@ -88,12 +172,16 @@ jobs: run: | cp config.json.example config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | flake8 + - name: Sort imports (isort) + run: | + isort --check . + - name: Mypy run: | mypy freqtrade scripts @@ -109,6 +197,7 @@ jobs: channel: '#notifications' url: ${{ secrets.SLACK_WEBHOOK }} + build_windows: runs-on: ${{ matrix.os }} @@ -121,7 +210,7 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} @@ -150,7 +239,7 @@ jobs: run: | cp config.json.example config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | @@ -172,7 +261,7 @@ jobs: url: ${{ secrets.SLACK_WEBHOOK }} docs_check: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 @@ -180,6 +269,17 @@ jobs: run: | ./tests/test_docs.sh + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Documentation build + run: | + pip install -r docs/requirements-docs.txt + pip install mkdocs + mkdocs build + - name: Slack Notification uses: homoluctus/slatify@v1.8.0 if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) @@ -190,18 +290,18 @@ jobs: url: ${{ secrets.SLACK_WEBHOOK }} cleanup-prior-runs: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Cleanup previous runs on this branch uses: rokroskar/workflow-run-cleanup-action@v0.2.2 - if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.repository == 'freqtrade/freqtrade'" + if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'" env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" # Notify on slack only once - when CI completes (and after deploy) in case it's successfull notify-complete: - needs: [ build, build_windows, docs_check ] - runs-on: ubuntu-latest + needs: [ build_linux, build_macos, build_windows, docs_check ] + runs-on: ubuntu-20.04 steps: - name: Slack Notification uses: homoluctus/slatify@v1.8.0 @@ -213,20 +313,21 @@ jobs: url: ${{ secrets.SLACK_WEBHOOK }} deploy: - needs: [ build, build_windows, docs_check ] - runs-on: ubuntu-18.04 + needs: [ build_linux, build_macos, build_windows, docs_check ] + runs-on: ubuntu-20.04 + if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: 3.8 - name: Extract branch name shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})" id: extract_branch - name: Build distribution @@ -236,7 +337,7 @@ jobs: - name: Publish to PyPI (Test) uses: pypa/gh-action-pypi-publish@master - if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release') + if: (github.event_name == 'release') with: user: __token__ password: ${{ secrets.pypi_test_password }} @@ -244,7 +345,7 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@master - if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release') + if: (github.event_name == 'release') with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/docker_update_readme.yml b/.github/workflows/docker_update_readme.yml index 57a7e591e..95e69be2a 100644 --- a/.github/workflows/docker_update_readme.yml +++ b/.github/workflows/docker_update_readme.yml @@ -2,7 +2,7 @@ name: Update Docker Hub Description on: push: branches: - - master + - stable jobs: dockerHubDescription: diff --git a/.readthedocs.yml b/.readthedocs.yml index dec7b44d7..446181452 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,5 +4,5 @@ build: image: latest python: - version: 3.6 - setup_py_install: false \ No newline at end of file + version: 3.8 + setup_py_install: false diff --git a/.travis.yml b/.travis.yml index 0cb76b78b..94239e33f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ os: - linux -dist: xenial +dist: bionic language: python python: -- 3.6 +- 3.8 services: - docker env: @@ -33,7 +33,7 @@ jobs: - script: - cp config.json.example config.json - freqtrade create-userdir --userdir user_data - - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt + - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily name: hyperopt - script: flake8 name: flake8 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 90594866a..6b4e8adaf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,17 +8,17 @@ Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/ Few pointers for contributions: -- Create your PR against the `develop` branch, not `master`. -- New features need to contain unit tests and must be PEP8 conformant (max-line-length = 100). +- Create your PR against the `develop` branch, not `stable`. +- New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR. +- PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished). -If you are unsure, discuss the feature on our [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) -or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. +If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. ## Getting started Best start by reading the [documentation](https://www.freqtrade.io/) to get a feel for what is possible with the bot, or head straight to the [Developer-documentation](https://www.freqtrade.io/en/latest/developer/) (WIP) which should help you getting started. -## Before sending the PR: +## Before sending the PR ### 1. Run unit tests @@ -64,6 +64,14 @@ Guide for installing them is [here](http://flake8.pycqa.org/en/latest/user/using mypy freqtrade ``` +### 4. Ensure all imports are correct + +#### Run isort + +``` bash +isort . +``` + ## (Core)-Committer Guide ### Process: Pull Requests @@ -114,6 +122,6 @@ Contributors may be given commit privileges. Preference will be given to those w 1. Access to resources for cross-platform development and testing. 1. Time to devote to the project regularly. -Being a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust Freqtrade with their Exchange API keys). +Being a Committer does not grant write permission on `develop` or `stable` for security reasons (Users trust Freqtrade with their Exchange API keys). After being Committer for some time, a Committer may be named Core Committer and given full repository access. diff --git a/Dockerfile b/Dockerfile index b6333fb13..602e6a28c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,46 @@ -FROM python:3.8.3-slim-buster +FROM python:3.8.6-slim-buster as base -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev \ - && apt-get clean \ - && pip install --upgrade pip +# Setup env +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONFAULTHANDLER 1 +ENV PATH=/root/.local/bin:$PATH # Prepare environment RUN mkdir /freqtrade WORKDIR /freqtrade +# Install dependencies +FROM base as python-deps +RUN apt-get update \ + && apt-get -y install curl build-essential libssl-dev git \ + && apt-get clean \ + && pip install --upgrade pip + # Install TA-lib COPY build_helpers/* /tmp/ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* - ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies -COPY requirements.txt requirements-common.txt requirements-hyperopt.txt /freqtrade/ -RUN pip install numpy --no-cache-dir \ - && pip install -r requirements-hyperopt.txt --no-cache-dir +COPY requirements.txt requirements-hyperopt.txt /freqtrade/ +RUN pip install --user --no-cache-dir numpy \ + && pip install --user --no-cache-dir -r requirements-hyperopt.txt + +# Copy dependencies to runtime-image +FROM base as runtime-image +COPY --from=python-deps /usr/local/lib /usr/local/lib +ENV LD_LIBRARY_PATH /usr/local/lib + +COPY --from=python-deps /root/.local /root/.local + + # Install and execute COPY . /freqtrade/ -RUN pip install -e . --no-cache-dir +RUN pip install -e . --no-cache-dir \ + && mkdir /freqtrade/user_data/ ENTRYPOINT ["freqtrade"] # Default to trade mode CMD [ "trade" ] diff --git a/Dockerfile.armhf b/Dockerfile.armhf index d6e2aa3a1..b6f2e44e6 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -1,25 +1,43 @@ -FROM --platform=linux/arm/v7 python:3.7.7-slim-buster +FROM --platform=linux/arm/v7 python:3.7.9-slim-buster as base -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev libatlas3-base libgfortran5 \ - && apt-get clean \ - && pip install --upgrade pip \ - && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf +# Setup env +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONFAULTHANDLER 1 +ENV PATH=/root/.local/bin:$PATH # Prepare environment RUN mkdir /freqtrade WORKDIR /freqtrade +RUN apt-get update \ + && apt-get -y install libatlas3-base curl sqlite3 \ + && apt-get clean + +# Install dependencies +FROM base as python-deps +RUN apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \ + && apt-get clean \ + && pip install --upgrade pip \ + && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf + # Install TA-lib COPY build_helpers/* /tmp/ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* - ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies -COPY requirements.txt requirements-common.txt /freqtrade/ -RUN pip install numpy --no-cache-dir \ - && pip install -r requirements.txt --no-cache-dir +COPY requirements.txt /freqtrade/ +RUN pip install --user --no-cache-dir numpy \ + && pip install --user --no-cache-dir -r requirements.txt + +# Copy dependencies to runtime-image +FROM base as runtime-image +COPY --from=python-deps /usr/local/lib /usr/local/lib +ENV LD_LIBRARY_PATH /usr/local/lib + +COPY --from=python-deps /root/.local /root/.local # Install and execute COPY . /freqtrade/ diff --git a/README.md b/README.md index cfb384702..a9aee342f 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Please find the complete documentation on our [website](https://www.freqtrade.io ## Features -- [x] **Based on Python 3.6+**: For botting on any operating system - Windows, macOS and Linux. +- [x] **Based on Python 3.7+**: For botting on any operating system - Windows, macOS and Linux. - [x] **Persistence**: Persistence is achieved through sqlite. - [x] **Dry-run**: Run the bot without playing money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. @@ -55,9 +55,8 @@ Please find the complete documentation on our [website](https://www.freqtrade.io Freqtrade provides a Linux/macOS script to install all dependencies and help you to configure the bot. ```bash -git clone git@github.com:freqtrade/freqtrade.git +git clone -b develop https://github.com/freqtrade/freqtrade.git cd freqtrade -git checkout develop ./setup.sh --install ``` @@ -82,7 +81,8 @@ positional arguments: new-hyperopt Create new hyperopt new-strategy Create new strategy download-data Download backtesting data. - convert-data Convert candle (OHLCV) data from one format to another. + convert-data Convert candle (OHLCV) data from one format to + another. convert-trade-data Convert trade data from one format to another. backtesting Backtesting module. edge Edge module. @@ -94,7 +94,7 @@ positional arguments: list-markets Print markets on exchange. list-pairs Print pairs on exchange. list-strategies Print available strategies. - list-timeframes Print available ticker intervals (timeframes) for the exchange. + list-timeframes Print available timeframes for the exchange. show-trades Show trades. test-pairlist Test your pairlist configuration. plot-dataframe Plot candles with indicators. @@ -110,35 +110,35 @@ optional arguments: Telegram is not mandatory. However, this is a great way to control your bot. More details and the full command list on our [documentation](https://www.freqtrade.io/en/latest/telegram-usage/) -- `/start`: Starts the trader -- `/stop`: Stops the trader -- `/status [table]`: Lists all open trades -- `/count`: Displays number of open trades +- `/start`: Starts the trader. +- `/stop`: Stops the trader. +- `/stopbuy`: Stop entering new trades. +- `/status [table]`: Lists all open trades. - `/profit`: Lists cumulative profit from all finished trades - `/forcesell |all`: Instantly sells the given trade (Ignoring `minimum_roi`). - `/performance`: Show performance of each finished trade grouped by pair -- `/balance`: Show account balance per currency -- `/daily `: Shows profit or loss per day, over the last n days -- `/help`: Show help message -- `/version`: Show version - +- `/balance`: Show account balance per currency. +- `/daily `: Shows profit or loss per day, over the last n days. +- `/help`: Show help message. +- `/version`: Show version. ## Development branches The project is currently setup in two main branches: -- `develop` - This branch has often new features, but might also cause breaking changes. -- `master` - This branch contains the latest stable release. The bot 'should' be stable on this branch, and is generally well tested. +- `develop` - This branch has often new features, but might also contain breaking changes. We try hard to keep this branch as stable as possible. +- `stable` - This branch contains the latest stable release. This branch is generally well tested. - `feat/*` - These are feature branches, which are being worked on heavily. Please don't use these unless you want to test a specific feature. ## Support -### Help / Slack +### Help / Discord / Slack -For any questions not covered by the documentation or for further -information about the bot, we encourage you to join our slack channel. +For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. -- [Click here to join Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE). +Please check out our [discord server](https://discord.gg/MA9v74M). + +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) @@ -166,18 +166,18 @@ Please read our [Contributing document](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) to understand the requirements before sending your pull-requests. -Coding is not a neccessity to contribute - maybe start with improving our documentation? +Coding is not a necessity to contribute - maybe start with improving our documentation? Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. -**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/MA9v74M) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. -**Important:** Always create your PR against the `develop` branch, not `master`. +**Important:** Always create your PR against the `develop` branch, not `stable`. ## Requirements -### Uptodate clock +### Up-to-date clock -The clock must be accurate, syncronized to a NTP server very frequently to avoid problems with communication to the exchanges. +The clock must be accurate, synchronized to a NTP server very frequently to avoid problems with communication to the exchanges. ### Min hardware required @@ -187,7 +187,7 @@ To run this bot we recommend you a cloud instance with a minimum of: ### Software requirements -- [Python 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/) +- [Python 3.7.x](http://docs.python-guide.org/en/latest/starting/installation/) - [pip](https://pip.pypa.io/en/stable/installing/) - [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - [TA-Lib](https://mrjbq7.github.io/ta-lib/install.html) diff --git a/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl deleted file mode 100644 index bd61e812b..000000000 Binary files a/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl deleted file mode 100644 index f81addb44..000000000 Binary files a/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl new file mode 100644 index 000000000..5adbda8a7 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.19-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.19-cp38-cp38-win_amd64.whl new file mode 100644 index 000000000..b652c7ee0 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.19-cp38-cp38-win_amd64.whl differ diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 index 0a55b6ddd..5747db335 100644 --- a/build_helpers/install_windows.ps1 +++ b/build_helpers/install_windows.ps1 @@ -7,10 +7,10 @@ python -m pip install --upgrade pip $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" if ($pyv -eq '3.7') { - pip install build_helpers\TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.19-cp37-cp37m-win_amd64.whl } if ($pyv -eq '3.8') { - pip install build_helpers\TA_Lib-0.4.18-cp38-cp38-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.19-cp38-cp38-win_amd64.whl } pip install -r requirements-dev.txt diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 03a95161b..ac0cd2461 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -2,6 +2,7 @@ # Replace / with _ to create a valid tag TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") +TAG_PLOT=${TAG}_plot echo "Running for ${TAG}" # Add commit and commit_message to docker container @@ -16,6 +17,12 @@ else docker pull ${IMAGE_NAME}:${TAG} docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . fi +# Tag image for upload and next build step +docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG + +docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot . + +docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT if [ $? -ne 0 ]; then echo "failed building image" @@ -30,8 +37,6 @@ if [ $? -ne 0 ]; then return 1 fi -# Tag image for upload -docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG if [ $? -ne 0 ]; then echo "failed tagging image" return 1 diff --git a/config.json.example b/config.json.example index ae87164d6..af45dac74 100644 --- a/config.json.example +++ b/config.json.example @@ -4,17 +4,16 @@ "stake_amount": 0.05, "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", - "ticker_interval": "5m", - "dry_run": false, + "timeframe": "5m", + "dry_run": true, "cancel_open_orders_on_exit": false, - "trailing_stop": false, "unfilledtimeout": { "buy": 10, "sell": 30 }, "bid_strategy": { - "ask_last_balance": 0.0, "use_order_book": false, + "ask_last_balance": 0.0, "order_book_top": 1, "check_depth_of_market": { "enabled": false, @@ -82,6 +81,7 @@ "listen_port": 8080, "verbosity": "info", "jwt_secret_key": "somethingrandom", + "CORS_origins": [], "username": "", "password": "" }, diff --git a/config_binance.json.example b/config_binance.json.example index 74207d565..f3f8eb659 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -4,10 +4,9 @@ "stake_amount": 0.05, "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", - "ticker_interval": "5m", + "timeframe": "5m", "dry_run": true, "cancel_open_orders_on_exit": false, - "trailing_stop": false, "unfilledtimeout": { "buy": 10, "sell": 30 @@ -87,6 +86,7 @@ "listen_port": 8080, "verbosity": "info", "jwt_secret_key": "somethingrandom", + "CORS_origins": [], "username": "", "password": "" }, diff --git a/config_full.json.example b/config_full.json.example index 57ebb22e6..e69e52469 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -7,9 +7,9 @@ "amount_reserve_percent": 0.05, "amend_last_stake_amount": false, "last_stake_amount_min_ratio": 0.5, - "dry_run": false, + "dry_run": true, "cancel_open_orders_on_exit": false, - "ticker_interval": "5m", + "timeframe": "5m", "trailing_stop": false, "trailing_stop_positive": 0.005, "trailing_stop_positive_offset": 0.0051, @@ -64,9 +64,43 @@ "sort_key": "quoteVolume", "refresh_period": 1800 }, + {"method": "AgeFilter", "min_days_listed": 10}, {"method": "PrecisionFilter"}, - {"method": "PriceFilter", "low_price_ratio": 0.01}, - {"method": "SpreadFilter", "max_spread_ratio": 0.005} + {"method": "PriceFilter", "low_price_ratio": 0.01, "min_price": 0.00000010}, + {"method": "SpreadFilter", "max_spread_ratio": 0.005}, + { + "method": "RangeStabilityFilter", + "lookback_days": 10, + "min_rate_of_change": 0.01, + "refresh_period": 1440 + } + ], + "protections": [ + { + "method": "StoplossGuard", + "lookback_period_candles": 60, + "trade_limit": 4, + "stop_duration_candles": 60, + "only_per_pair": false + }, + { + "method": "CooldownPeriod", + "stop_duration_candles": 20 + }, + { + "method": "MaxDrawdown", + "lookback_period_candles": 200, + "trade_limit": 20, + "stop_duration_candles": 10, + "max_allowed_drawdown": 0.2 + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 360, + "trade_limit": 1, + "stop_duration_candles": 2, + "required_profit": 0.02 + } ], "exchange": { "name": "bittrex", @@ -115,7 +149,16 @@ "telegram": { "enabled": true, "token": "your_telegram_token", - "chat_id": "your_telegram_chat_id" + "chat_id": "your_telegram_chat_id", + "notification_settings": { + "status": "on", + "warning": "on", + "startup": "on", + "buy": "on", + "sell": "on", + "buy_cancel": "on", + "sell_cancel": "on" + } }, "api_server": { "enabled": false, @@ -123,6 +166,7 @@ "listen_port": 8080, "verbosity": "info", "jwt_secret_key": "somethingrandom", + "CORS_origins": [], "username": "freqtrader", "password": "SuperSecurePassword" }, diff --git a/config_kraken.json.example b/config_kraken.json.example index f1e522da5..5f3b57854 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -4,10 +4,9 @@ "stake_amount": 10, "tradable_balance_ratio": 0.99, "fiat_display_currency": "EUR", - "ticker_interval": "5m", + "timeframe": "5m", "dry_run": true, "cancel_open_orders_on_exit": false, - "trailing_stop": false, "unfilledtimeout": { "buy": 10, "sell": 30 @@ -28,12 +27,11 @@ "use_sell_signal": true, "sell_profit_only": false, "ignore_roi_if_buy_signal": false - }, "exchange": { "name": "kraken", - "key": "", - "secret": "", + "key": "your_exchange_key", + "secret": "your_exchange_key", "ccxt_config": {"enableRateLimit": true}, "ccxt_async_config": { "enableRateLimit": true, @@ -93,6 +91,7 @@ "listen_port": 8080, "verbosity": "info", "jwt_secret_key": "somethingrandom", + "CORS_origins": [], "username": "", "password": "" }, diff --git a/docker-compose.develop.yml b/docker-compose.develop.yml deleted file mode 100644 index 562b5960a..000000000 --- a/docker-compose.develop.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -version: '3' -services: - freqtrade_develop: - build: - context: . - dockerfile: "./Dockerfile.develop" - volumes: - - ".:/freqtrade" - entrypoint: - - "freqtrade" - - freqtrade_bash: - build: - context: . - dockerfile: "./Dockerfile.develop" - volumes: - - ".:/freqtrade" - entrypoint: - - "/bin/bash" diff --git a/docker-compose.yml b/docker-compose.yml index 49d83aa5e..7094500b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,12 +2,14 @@ version: '3' services: freqtrade: - image: freqtradeorg/freqtrade:master + image: freqtradeorg/freqtrade:stable # image: freqtradeorg/freqtrade:develop + # Use plotting image + # image: freqtradeorg/freqtrade:develop_plot # Build step - only needed when additional dependencies are needed # build: # context: . - # dockerfile: "./Dockerfile.technical" + # dockerfile: "./docker/Dockerfile.technical" restart: unless-stopped container_name: freqtrade volumes: diff --git a/Dockerfile.develop b/docker/Dockerfile.develop similarity index 99% rename from Dockerfile.develop rename to docker/Dockerfile.develop index 8f6871c55..cb49984e2 100644 --- a/Dockerfile.develop +++ b/docker/Dockerfile.develop @@ -2,6 +2,7 @@ FROM freqtradeorg/freqtrade:develop # Install dependencies COPY requirements-dev.txt /freqtrade/ + RUN pip install numpy --no-cache-dir \ && pip install -r requirements-dev.txt --no-cache-dir diff --git a/docker/Dockerfile.jupyter b/docker/Dockerfile.jupyter new file mode 100644 index 000000000..b7499eeef --- /dev/null +++ b/docker/Dockerfile.jupyter @@ -0,0 +1,7 @@ +FROM freqtradeorg/freqtrade:develop_plot + + +RUN pip install jupyterlab --no-cache-dir + +# Empty the ENTRYPOINT to allow all commands +ENTRYPOINT [] diff --git a/docker/Dockerfile.plot b/docker/Dockerfile.plot new file mode 100644 index 000000000..40bc72bc5 --- /dev/null +++ b/docker/Dockerfile.plot @@ -0,0 +1,7 @@ +ARG sourceimage=develop +FROM freqtradeorg/freqtrade:${sourceimage} + +# Install dependencies +COPY requirements-plot.txt /freqtrade/ + +RUN pip install -r requirements-plot.txt --no-cache-dir diff --git a/Dockerfile.technical b/docker/Dockerfile.technical similarity index 100% rename from Dockerfile.technical rename to docker/Dockerfile.technical diff --git a/docker/docker-compose-jupyter.yml b/docker/docker-compose-jupyter.yml new file mode 100644 index 000000000..11a01705c --- /dev/null +++ b/docker/docker-compose-jupyter.yml @@ -0,0 +1,16 @@ +--- +version: '3' +services: + ft_jupyterlab: + build: + context: .. + dockerfile: docker/Dockerfile.jupyter + restart: unless-stopped + container_name: freqtrade + ports: + - "127.0.0.1:8888:8888" + volumes: + - "./user_data:/freqtrade/user_data" + # Default command used when running `docker compose up` + command: > + jupyter lab --port=8888 --ip 0.0.0.0 --allow-root diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 5fc674b03..1ace61769 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -27,9 +27,9 @@ class MyAwesomeHyperOpt2(MyAwesomeHyperOpt): and then quickly switch between hyperopt classes, running optimization process with hyperopt class you need in each particular case: ``` -$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt ... +$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy ... or -$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt2 ... +$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt2 --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy ... ``` ## Creating and using a custom loss function @@ -77,7 +77,7 @@ Currently, the arguments are: * `results`: DataFrame containing the result The following columns are available in results (corresponds to the output-file of backtesting when used with `--export trades`): - `pair, profit_percent, profit_abs, open_time, close_time, open_index, close_index, trade_duration, open_at_end, open_rate, close_rate, sell_reason` + `pair, profit_percent, profit_abs, open_date, open_rate, open_fee, close_date, close_rate, close_fee, amount, trade_duration, open_at_end, sell_reason` * `trade_count`: Amount of trades (identical to `len(results)`) * `min_date`: Start date of the hyperopting TimeFrame * `min_date`: End date of the hyperopting TimeFrame diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md index 95480a2c6..f03bc10c0 100644 --- a/docs/advanced-setup.md +++ b/docs/advanced-setup.md @@ -4,6 +4,54 @@ This page explains some advanced tasks and configuration options that can be per If you do not know what things mentioned here mean, you probably do not need it. +## Running multiple instances of Freqtrade + +This section will show you how to run multiple bots at the same time, on the same machine. + +### Things to consider + +* Use different database files. +* Use different Telegram bots (requires multiple different configuration files; applies only when Telegram is enabled). +* Use different ports (applies only when Freqtrade REST API webserver is enabled). + +### Different database files + +In order to keep track of your trades, profits, etc., freqtrade is using a SQLite database where it stores various types of information such as the trades you performed in the past and the current position(s) you are holding at any time. This allows you to keep track of your profits, but most importantly, keep track of ongoing activity if the bot process would be restarted or would be terminated unexpectedly. + +Freqtrade will, by default, use separate database files for dry-run and live bots (this assumes no database-url is given in either configuration nor via command line argument). +For live trading mode, the default database will be `tradesv3.sqlite` and for dry-run it will be `tradesv3.dryrun.sqlite`. + +The optional argument to the trade command used to specify the path of these files is `--db-url`, which requires a valid SQLAlchemy url. +So when you are starting a bot with only the config and strategy arguments in dry-run mode, the following 2 commands would have the same outcome. + +``` bash +freqtrade trade -c MyConfig.json -s MyStrategy +# is equivalent to +freqtrade trade -c MyConfig.json -s MyStrategy --db-url sqlite:///tradesv3.dryrun.sqlite +``` + +It means that if you are running the trade command in two different terminals, for example to test your strategy both for trades in USDT and in another instance for trades in BTC, you will have to run them with different databases. + +If you specify the URL of a database which does not exist, freqtrade will create one with the name you specified. So to test your custom strategy with BTC and USDT stake currencies, you could use the following commands (in 2 separate terminals): + +``` bash +# Terminal 1: +freqtrade trade -c MyConfigBTC.json -s MyCustomStrategy --db-url sqlite:///user_data/tradesBTC.dryrun.sqlite +# Terminal 2: +freqtrade trade -c MyConfigUSDT.json -s MyCustomStrategy --db-url sqlite:///user_data/tradesUSDT.dryrun.sqlite +``` + +Conversely, if you wish to do the same thing in production mode, you will also have to create at least one new database (in addition to the default one) and specify the path to the "live" databases, for example: + +``` bash +# Terminal 1: +freqtrade trade -c MyConfigBTC.json -s MyCustomStrategy --db-url sqlite:///user_data/tradesBTC.live.sqlite +# Terminal 2: +freqtrade trade -c MyConfigUSDT.json -s MyCustomStrategy --db-url sqlite:///user_data/tradesUSDT.live.sqlite +``` + +For more information regarding usage of the sqlite databases, for example to manually enter or remove trades, please refer to the [SQL Cheatsheet](sql_cheatsheet.md). + ## Configure the bot running as a systemd service Copy the `freqtrade.service` file to your systemd user directory (usually `~/.config/systemd/user`) and update `WorkingDirectory` and `ExecStart` to match your setup. diff --git a/docs/backtesting.md b/docs/backtesting.md index 9b2997510..27bfebe37 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -12,7 +12,7 @@ real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHCLV) data from `user_data/data/` by default. -If no data is available for the exchange / pair / timeframe (ticker interval) combination, backtesting will ask you to download them first using `freqtrade download-data`. +If no data is available for the exchange / pair / timeframe combination, backtesting will ask you to download them first using `freqtrade download-data`. For details on downloading, please refer to the [Data Downloading](data-download.md) section in the documentation. The result of backtesting will confirm if your bot has better odds of making a profit than a loss. @@ -35,7 +35,7 @@ freqtrade backtesting #### With 1 min candle (OHLCV) data ```bash -freqtrade backtesting --ticker-interval 1m +freqtrade backtesting --timeframe 1m ``` #### Using a different on-disk historical candle (OHLCV) data source @@ -58,7 +58,7 @@ Where `-s SampleStrategy` refers to the class name within the strategy file `sam #### Comparing multiple Strategies ```bash -freqtrade backtesting --strategy-list SampleStrategy1 AwesomeStrategy --ticker-interval 5m +freqtrade backtesting --strategy-list SampleStrategy1 AwesomeStrategy --timeframe 5m ``` Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies. @@ -66,7 +66,7 @@ Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies #### Exporting trades to file ```bash -freqtrade backtesting --export trades +freqtrade backtesting --export trades --config config.json --strategy SampleStrategy ``` The exported trades can be used for [further analysis](#further-backtest-result-analysis), or can be used by the plotting script `plot_dataframe.py` in the scripts directory. @@ -157,17 +157,37 @@ A backtesting result will look like that: | ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 | 0 | 0 | | LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 | 0 | 0 | | TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 | 0 | 0 | +=============== SUMMARY METRICS =============== +| Metric | Value | +|-----------------------+---------------------| +| Backtesting from | 2019-01-01 00:00:00 | +| Backtesting to | 2019-05-01 00:00:00 | +| Max open trades | 3 | +| | | +| Total trades | 429 | +| Total Profit % | 152.41% | +| Trades per day | 3.575 | +| | | +| Best Pair | LSK/BTC 26.26% | +| Worst Pair | ZEC/BTC -10.18% | +| Best Trade | LSK/BTC 4.25% | +| Worst Trade | ZEC/BTC -10.25% | +| Best day | 25.27% | +| Worst day | -30.67% | +| Avg. Duration Winners | 4:23:00 | +| Avg. Duration Loser | 6:55:00 | +| | | +| Max Drawdown | 50.63% | +| Drawdown Start | 2019-02-15 14:10:00 | +| Drawdown End | 2019-04-11 18:15:00 | +| Market change | -5.88% | +=============================================== ``` +### Backtesting report table + The 1st table contains all trades the bot made, including "left open trades". -The 2nd table contains a recap of sell reasons. -This table can tell you which area needs some additional work (i.e. all `sell_signal` trades are losses, so we should disable the sell-signal or work on improving that). - -The 3rd table contains all trades the bot had to `forcesell` at the end of the backtest period to present a full picture. -This is necessary to simulate realistic behaviour, since the backtest period has to end at some point, while realistically, you could leave the bot running forever. -These trades are also included in the first table, but are extracted separately for clarity. - The last line will give you the overall performance of your strategy, here: @@ -196,23 +216,87 @@ On the other hand, if you set a too high `minimal_roi` like `"0": 0.55` (55%), there is almost no chance that the bot will ever reach this profit. Hence, keep in mind that your performance is an integral mix of all different elements of the strategy, your configuration, and the crypto-currency pairs you have set up. +### Sell reasons table + +The 2nd table contains a recap of sell reasons. +This table can tell you which area needs some additional work (e.g. all or many of the `sell_signal` trades are losses, so you should work on improving the sell signal, or consider disabling it). + +### Left open trades table + +The 3rd table contains all trades the bot had to `forcesell` at the end of the backtesting period to present you the full picture. +This is necessary to simulate realistic behavior, since the backtest period has to end at some point, while realistically, you could leave the bot running forever. +These trades are also included in the first table, but are also shown separately in this table for clarity. + +### Summary metrics + +The last element of the backtest report is the summary metrics table. +It contains some useful key metrics about performance of your strategy on backtesting data. + +``` +=============== SUMMARY METRICS =============== +| Metric | Value | +|-----------------------+---------------------| +| Backtesting from | 2019-01-01 00:00:00 | +| Backtesting to | 2019-05-01 00:00:00 | +| Max open trades | 3 | +| | | +| Total trades | 429 | +| Total Profit % | 152.41% | +| Trades per day | 3.575 | +| | | +| Best Pair | LSK/BTC 26.26% | +| Worst Pair | ZEC/BTC -10.18% | +| Best Trade | LSK/BTC 4.25% | +| Worst Trade | ZEC/BTC -10.25% | +| Best day | 25.27% | +| Worst day | -30.67% | +| Avg. Duration Winners | 4:23:00 | +| Avg. Duration Loser | 6:55:00 | +| | | +| Max Drawdown | 50.63% | +| Drawdown Start | 2019-02-15 14:10:00 | +| Drawdown End | 2019-04-11 18:15:00 | +| Market change | -5.88% | +=============================================== + +``` + +- `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option). +- `Max open trades`: Setting of `max_open_trades` (or `--max-open-trades`) - to clearly see settings for this. +- `Total trades`: Identical to the total trades of the backtest output table. +- `Total Profit %`: Total profit per stake amount. Aligned to the TOTAL column of the first table. +- `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). +- `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. +- `Best Trade` / `Worst Trade`: Biggest winning trade and biggest losing trade +- `Best day` / `Worst day`: Best and worst day based on daily profit. +- `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades. +- `Max Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced). +- `Drawdown Start` / `Drawdown End`: Start and end datetime for this largest drawdown (can also be visualized via the `plot-dataframe` sub-command). +- `Market change`: Change of the market during the backtest period. Calculated as average of all pairs changes from the first to the last candle using the "close" column. + ### Assumptions made by backtesting Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: - Buys happen at open-price -- Sell signal sells happen at open-price of the following candle -- Low happens before high for stoploss, protecting capital first +- Sell-signal sells happen at open-price of the consecutive candle +- Sell-signal is favored over Stoploss, because sell-signals are assumed to trigger on candle's open - ROI - sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%) - sells are never "below the candle", so a ROI of 2% may result in a sell at 2.4% if low was at 2.4% profit - Forcesells caused by `=-1` ROI entries use low as sell value, unless N falls on the candle open (e.g. `120: -1` for 1h candles) -- Stoploss sells happen exactly at stoploss price, even if low was lower +- Stoploss sells happen exactly at stoploss price, even if low was lower, but the loss will be `2 * fees` higher than the stoploss price +- Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes +- Low happens before high for stoploss, protecting capital first - Trailing stoploss - High happens first - adjusting stoploss - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly) + - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies - Sell-reason does not explain if a trade was positive or negative, just what triggered the sell (this can look odd if negative ROI values are used) -- Stoploss (and trailing stoploss) is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` and/or `trailing_stop` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes. +- Evaluation sequence (if multiple signals happen on the same candle) + - ROI (if not stoploss) + - Sell-signal + - Stoploss Taking these assumptions, backtesting tries to mirror real trading as closely as possible. However, backtesting will **never** replace running a strategy in dry-run mode. Also, keep in mind that past results don't guarantee future success. @@ -228,13 +312,13 @@ You can then load the trades to perform further analysis as shown in our [data a To compare multiple strategies, a list of Strategies can be provided to backtesting. -This is limited to 1 timeframe (ticker interval) value per run. However, data is only loaded once from disk so if you have multiple +This is limited to 1 timeframe value per run. However, data is only loaded once from disk so if you have multiple strategies you'd like to compare, this will give a nice runtime boost. All listed Strategies need to be in the same directory. ``` bash -freqtrade backtesting --timerange 20180401-20180410 --ticker-interval 5m --strategy-list Strategy001 Strategy002 --export trades +freqtrade backtesting --timerange 20180401-20180410 --timeframe 5m --strategy-list Strategy001 Strategy002 --export trades ``` This will save the results to `user_data/backtest_results/backtest-result-.json`, injecting the strategy-name into the target filename. diff --git a/docs/bot-basics.md b/docs/bot-basics.md new file mode 100644 index 000000000..44f493456 --- /dev/null +++ b/docs/bot-basics.md @@ -0,0 +1,58 @@ +# Freqtrade basics + +This page provides you some basic concepts on how Freqtrade works and operates. + +## Freqtrade terminology + +* Trade: Open position. +* Open Order: Order which is currently placed on the exchange, and is not yet complete. +* Pair: Tradable pair, usually in the format of Quote/Base (e.g. XRP/USDT). +* Timeframe: Candle length to use (e.g. `"5m"`, `"1h"`, ...). +* Indicators: Technical indicators (SMA, EMA, RSI, ...). +* Limit order: Limit orders which execute at the defined limit price or better. +* Market order: Guaranteed to fill, may move price depending on the order size. + +## Fee handling + +All profit calculations of Freqtrade include fees. For Backtesting / Hyperopt / Dry-run modes, the exchange default fee is used (lowest tier on the exchange). For live operations, fees are used as applied by the exchange (this includes BNB rebates etc.). + +## Bot execution logic + +Starting freqtrade in dry-run or live mode (using `freqtrade trade`) will start the bot and start the bot iteration loop. +By default, loop runs every few seconds (`internals.process_throttle_secs`) and does roughly the following in the following sequence: + +* Fetch open trades from persistence. +* Calculate current list of tradable pairs. +* Download ohlcv data for the pairlist including all [informative pairs](strategy-customization.md#get-data-for-non-tradeable-pairs) + This step is only executed once per Candle to avoid unnecessary network traffic. +* Call `bot_loop_start()` strategy callback. +* Analyze strategy per pair. + * Call `populate_indicators()` + * Call `populate_buy_trend()` + * Call `populate_sell_trend()` +* Check timeouts for open orders. + * Calls `check_buy_timeout()` strategy callback for open buy orders. + * Calls `check_sell_timeout()` strategy callback for open sell orders. +* Verifies existing positions and eventually places sell orders. + * Considers stoploss, ROI and sell-signal. + * Determine sell-price based on `ask_strategy` configuration setting. + * Before a sell order is placed, `confirm_trade_exit()` strategy callback is called. +* Check if trade-slots are still available (if `max_open_trades` is reached). +* Verifies buy signal trying to enter new positions. + * Determine buy-price based on `bid_strategy` configuration setting. + * Before a buy order is placed, `confirm_trade_entry()` strategy callback is called. + +This loop will be repeated again and again until the bot is stopped. + +## Backtesting / Hyperopt execution logic + +[backtesting](backtesting.md) or [hyperopt](hyperopt.md) do only part of the above logic, since most of the trading operations are fully simulated. + +* Load historic data for configured pairlist. +* Calculate indicators (calls `populate_indicators()`). +* Calls `populate_buy_trend()` and `populate_sell_trend()` +* Loops per candle simulating entry and exit points. +* Generate backtest report output + +!!! Note + Both Backtesting and Hyperopt include exchange default Fees in the calculation. Custom fees can be passed to backtesting / hyperopt by specifying the `--fee` argument. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index b1649374a..5820b3cc7 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -5,26 +5,42 @@ This page explains the different parameters of the bot and how to run it. !!! Note If you've used `setup.sh`, don't forget to activate your virtual environment (`source .env/bin/activate`) before running freqtrade commands. +!!! Warning "Up-to-date clock" + The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges. + ## Bot commands ``` usage: freqtrade [-h] [-V] - {trade,backtesting,edge,hyperopt,create-userdir,list-exchanges,list-timeframes,download-data,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} ... Free, open source crypto trading bot positional arguments: - {trade,backtesting,edge,hyperopt,create-userdir,list-exchanges,list-timeframes,download-data,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} trade Trade module. + create-userdir Create user-data directory. + new-config Create new config + new-hyperopt Create new hyperopt + new-strategy Create new strategy + download-data Download backtesting data. + convert-data Convert candle (OHLCV) data from one format to + another. + convert-trade-data Convert trade data from one format to another. backtesting Backtesting module. edge Edge module. hyperopt Hyperopt module. - create-userdir Create user-data directory. + hyperopt-list List Hyperopt results + hyperopt-show Show details of Hyperopt results list-exchanges Print available exchanges. - list-timeframes Print available ticker intervals (timeframes) for the - exchange. - download-data Download backtesting data. + list-hyperopts Print available hyperopt classes. + list-markets Print markets on exchange. + list-pairs Print pairs on exchange. + list-strategies Print available strategies. + list-timeframes Print available timeframes for the exchange. + show-trades Show trades. + test-pairlist Test your pairlist configuration. plot-dataframe Plot candles with indicators. plot-profit Generate plot showing profits. @@ -72,7 +88,6 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. -. ``` @@ -197,20 +212,25 @@ Backtesting also uses the config specified via `-c/--config`. ``` usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] - [--strategy-path PATH] [-i TICKER_INTERVAL] - [--timerange TIMERANGE] [--max-open-trades INT] + [--strategy-path PATH] [-i TIMEFRAME] + [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] + [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] - [--eps] [--dmmp] + [--eps] [--dmmp] [--enable-protections] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export EXPORT] [--export-filename PATH] optional arguments: -h, --help show this help message and exit - -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`). --timerange TIMERANGE Specify what timerange of data to use. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `None`). --max-open-trades INT Override the value of the `max_open_trades` configuration setting. @@ -226,6 +246,10 @@ optional arguments: Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number). + --enable-protections, --enableprotections + Enable protections for backtesting.Will slow + backtesting down by a considerable amount, but will + include configured protections --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] Provide a space-separated list of strategies to backtest. Please note that ticker-interval needs to be @@ -280,23 +304,27 @@ to find optimal parameter values for your strategy. ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] - [-i TICKER_INTERVAL] [--timerange TIMERANGE] + [-i TIMEFRAME] [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--hyperopt NAME] [--hyperopt-path PATH] [--eps] - [-e INT] + [--dmmp] [--enable-protections] [-e INT] [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]] - [--dmmp] [--print-all] [--no-color] [--print-json] - [-j JOBS] [--random-state INT] [--min-trades INT] - [--continue] [--hyperopt-loss NAME] + [--print-all] [--no-color] [--print-json] [-j JOBS] + [--random-state INT] [--min-trades INT] + [--hyperopt-loss NAME] optional arguments: -h, --help show this help message and exit - -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`). --timerange TIMERANGE Specify what timerange of data to use. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `None`). --max-open-trades INT Override the value of the `max_open_trades` configuration setting. @@ -312,18 +340,22 @@ optional arguments: --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). - -e INT, --epochs INT Specify number of epochs (default: 100). - --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...] - Specify which parameters to hyperopt. Space-separated - list. --dmmp, --disable-max-market-positions Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number). + --enable-protections, --enableprotections + Enable protections for backtesting.Will slow + backtesting down by a considerable amount, but will + include configured protections + -e INT, --epochs INT Specify number of epochs (default: 100). + --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...] + Specify which parameters to hyperopt. Space-separated + list. --print-all Print all results, not only the best ones. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. - --print-json Print best results in JSON format. + --print-json Print output in JSON format. -j JOBS, --job-workers JOBS The number of concurrently running jobs for hyperoptimization (hyperopt worker processes). If -1 @@ -334,18 +366,14 @@ optional arguments: reproducible hyperopt results. --min-trades INT Set minimal desired number of trades for evaluations in the hyperopt optimization path (default: 1). - --continue Continue hyperopt from previous runs. By default, - temporary files will be removed and hyperopt will - start from scratch. --hyperopt-loss NAME Specify the class name of the hyperopt loss function class (IHyperOptLoss). Different functions can generate completely different results, since the target for optimization is different. Built-in Hyperopt-loss-functions are: - DefaultHyperOptLoss, OnlyProfitHyperOptLoss, + ShortTradeDurHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, - SortinoHyperOptLoss, SortinoHyperOptLossDaily. - (default: `DefaultHyperOptLoss`). + SortinoHyperOptLoss, SortinoHyperOptLossDaily Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -378,13 +406,13 @@ To know your trade expectancy and winrate against historical data, you can use E ``` usage: freqtrade edge [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] - [-i TICKER_INTERVAL] [--timerange TIMERANGE] + [-i TIMEFRAME] [--timerange TIMERANGE] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--stoplosses STOPLOSS_RANGE] optional arguments: -h, --help show this help message and exit - -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`). --timerange TIMERANGE diff --git a/docs/configuration.md b/docs/configuration.md index da0f015e8..b70a85c04 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,7 +47,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `amend_last_stake_amount` | Use reduced last stake amount if necessary. [More information below](#configuring-amount-per-trade).
*Defaults to `false`.*
**Datatype:** Boolean | `last_stake_amount_min_ratio` | Defines minimum stake amount that has to be left and executed. Applies only to the last stake amount when it's amended to a reduced value (i.e. if `amend_last_stake_amount` is set to `true`). [More information below](#configuring-amount-per-trade).
*Defaults to `0.5`.*
**Datatype:** Float (as ratio) | `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals.
*Defaults to `0.05` (5%).*
**Datatype:** Positive Float as ratio. -| `ticker_interval` | The timeframe (ticker interval) to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String +| `timeframe` | The timeframe (former ticker interval) to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String | `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
**Datatype:** String | `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
**Datatype:** Boolean | `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in the Dry Run mode.
*Defaults to `1000`.*
**Datatype:** Float @@ -55,12 +55,12 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean | `minimal_roi` | **Required.** Set the threshold as ratio the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict | `stoploss` | **Required.** Value as ratio of the stoploss used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Float (as ratio) -| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Boolean -| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Float -| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
**Datatype:** Float +| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md#trailing-stop-loss). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Boolean +| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-custom-positive-loss). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Float +| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
**Datatype:** Float | `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean -| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer -| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer +| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer +| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer | `bid_strategy.price_side` | Select the side of the spread the bot should look at to get the buy rate. [More information below](#buy-price-side).
*Defaults to `bid`.*
**Datatype:** String (either `ask` or `bid`). | `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#buy-price-without-orderbook-enabled). | `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled).
**Datatype:** Boolean @@ -87,9 +87,11 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `exchange.ccxt_sync_config` | Additional CCXT parameters passed to the regular (sync) ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict | `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
**Datatype:** Positive Integer +| `exchange.skip_pair_validation` | Skip pairlist validation on startup.
*Defaults to `false`
**Datatype:** Boolean | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean | `pairlists` | Define one or more pairlists to be used. [More information below](#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts +| `protections` | Define one or more protections to be used. [More information below](#protections).
**Datatype:** List of Dicts | `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String @@ -126,7 +128,7 @@ The following parameters can be set in either configuration file or strategy. Values set in the configuration file always overwrite values set in the strategy. * `minimal_roi` -* `ticker_interval` +* `timeframe` * `stoploss` * `trailing_stop` * `trailing_stop_positive` @@ -176,7 +178,7 @@ In the example above this would mean: This option only applies with [Static stake amount](#static-stake-amount) - since [Dynamic stake amount](#dynamic-stake-amount) divides the balances evenly. !!! Note - The minimum last stake amount can be configured using `amend_last_stake_amount` - which defaults to 0.5 (50%). This means that the minimum stake amount that's ever used is `stake_amount * 0.5`. This avoids very low stake amounts, that are close to the minimum tradable amount for the pair and can be refused by the exchange. + The minimum last stake amount can be configured using `last_stake_amount_min_ratio` - which defaults to 0.5 (50%). This means that the minimum stake amount that's ever used is `stake_amount * 0.5`. This avoids very low stake amounts, that are close to the minimum tradable amount for the pair and can be refused by the exchange. #### Static stake amount @@ -272,26 +274,19 @@ the static list of pairs) if we should buy. ### Understand order_types -The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. +The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`, `emergencysell`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. This allows to buy using limit orders, sell using -limit-orders, and create stoplosses using using market orders. It also allows to set the +limit-orders, and create stoplosses using market orders. It also allows to set the stoploss "on exchange" which means stoploss order would be placed immediately once the buy order is fulfilled. -If `stoploss_on_exchange` and `trailing_stop` are both set, then the bot will use `stoploss_on_exchange_interval` to check and update the stoploss on exchange periodically. -`order_types` can be set in the configuration file or in the strategy. + `order_types` set in the configuration file overwrites values set in the strategy as a whole, so you need to configure the whole `order_types` dictionary in one place. If this is configured, the following 4 values (`buy`, `sell`, `stoploss` and `stoploss_on_exchange`) need to be present, otherwise the bot will fail to start. -`emergencysell` is an optional value, which defaults to `market` and is used when creating stoploss on exchange orders fails. -The below is the default which is used if this is not configured in either strategy or configuration file. - -Since `stoploss_on_exchange` uses limit orders, the exchange needs 2 prices, the stoploss_price and the Limit price. -`stoploss` defines the stop-price - and limit should be slightly below this. This defaults to 0.99 / 1% (configurable via `stoploss_on_exchange_limit_ratio`). -Calculation example: we bought the asset at 100$. -Stop-price is 95$, then limit would be `95 * 0.99 = 94.05$` - so the stoploss will happen between 95$ and 94.05$. +For information on (`emergencysell`,`stoploss_on_exchange`,`stoploss_on_exchange_interval`,`stoploss_on_exchange_limit_ratio`) please see stop loss documentation [stop loss on exchange](stoploss.md) Syntax for Strategy: @@ -320,18 +315,20 @@ Configuration: } ``` -!!! Note +!!! Note "Market order support" Not all exchanges support "market" orders. The following message will be shown if your exchange does not support market orders: - `"Exchange does not support market orders."` + `"Exchange does not support market orders."` and the bot will refuse to start. -!!! Note - Stoploss on exchange interval is not mandatory. Do not change its value if you are +!!! Warning "Using market orders" + Please carefully read the section [Market order pricing](#market-order-pricing) section when using market orders. + +!!! Note "Stoploss on exchange" + `stoploss_on_exchange_interval` is not mandatory. Do not change its value if you are unsure of what you are doing. For more information about how stoploss works please refer to [the stoploss documentation](stoploss.md). -!!! Note - If `stoploss_on_exchange` is enabled and the stoploss is cancelled manually on the exchange, then the bot will create a new order. + If `stoploss_on_exchange` is enabled and the stoploss is cancelled manually on the exchange, then the bot will create a new stoploss order. !!! Warning "Warning: stoploss_on_exchange failures" If stoploss on exchange creation fails for some reason, then an "emergency sell" is initiated. By default, this will sell the asset using a market order. The order-type for the emergency-sell can be changed by setting the `emergencysell` value in the `order_types` dictionary - however this is not advised. @@ -379,7 +376,7 @@ Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports exchange markets and trading APIs. The complete up-to-date list can be found in the [CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested by the development team with only Bittrex, Binance and Kraken, - so the these are the only officially supported exhanges: + so the these are the only officially supported exchanges: - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" @@ -459,6 +456,9 @@ Prices are always retrieved right before an order is placed, either by querying !!! Note Orderbook data used by Freqtrade are the data retrieved from exchange by the ccxt's function `fetch_order_book()`, i.e. are usually data from the L2-aggregated orderbook, while the ticker data are the structures returned by the ccxt's `fetch_ticker()`/`fetch_tickers()` functions. Refer to the ccxt library [documentation](https://github.com/ccxt/ccxt/wiki/Manual#market-data) for more details. +!!! Warning "Using market orders" + Please read the section [Market order pricing](#market-order-pricing) section when using market orders. + ### Buy price #### Check depth of market @@ -553,118 +553,30 @@ A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting When not using orderbook (`ask_strategy.use_order_book=False`), the price at the `ask_strategy.price_side` side (defaults to `"ask"`) from the ticker will be used as the sell price. -## Pairlists and Pairlist Handlers +### Market order pricing -Pairlist Handlers define the list of pairs (pairlist) that the bot should trade. They are configured in the `pairlists` section of the configuration settings. +When using market orders, prices should be configured to use the "correct" side of the orderbook to allow realistic pricing detection. +Assuming both buy and sell are using market orders, a configuration similar to the following might be used -In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). - -Additionaly, [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. - -If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler. - -Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist. - -### Available Pairlist Handlers - -* [`StaticPairList`](#static-pair-list) (default, if not configured differently) -* [`VolumePairList`](#volume-pair-list) -* [`PrecisionFilter`](#precisionfilter) -* [`PriceFilter`](#pricefilter) -* [`ShuffleFilter`](#shufflefilter) -* [`SpreadFilter`](#spreadfilter) - -!!! Tip "Testing pairlists" - Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility subcommand to test your configuration quickly. - -#### Static Pair List - -By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. - -It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. - -```json -"pairlists": [ - {"method": "StaticPairList"} - ], +``` jsonc + "order_types": { + "buy": "market", + "sell": "market" + // ... + }, + "bid_strategy": { + "price_side": "ask", + // ... + }, + "ask_strategy":{ + "price_side": "bid", + // ... + }, ``` -#### Volume Pair List - -`VolumePairList` employs sorting/filtering of pairs by their trading volume. I selects `number_assets` top pairs with sorting based on the `sort_key` (which can only be `quoteVolume`). - -When used in the chain of Pairlist Handlers in a non-leading position (after StaticPairList and other Pairlist Filters), `VolumePairList` considers outputs of previous Pairlist Handlers, adding its sorting/selection of the pairs by the trading volume. - -When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top assets from all available markets (with matching stake-currency) on the exchange. - -The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). - -`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library: - -* The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours. - -```json -"pairlists": [{ - "method": "VolumePairList", - "number_assets": 20, - "sort_key": "quoteVolume", - "refresh_period": 1800, -], -``` - -#### PrecisionFilter - -Filters low-value coins which would not allow setting stoplosses. - -#### PriceFilter - -The `PriceFilter` allows filtering of pairs by price. - -Currently, only `low_price_ratio` setting is implemented, where a raise of 1 price unit (pip) is below the `low_price_ratio` ratio. -This option is disabled by default, and will only apply if set to <> 0. - -Calculation example: - -Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0.00000012 - which is almost 10% higher than the previous value. - -These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses. Here is what the PriceFilters takes over. - -#### ShuffleFilter - -Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. - -!!! Tip - You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. - -#### SpreadFilter - -Removes pairs that have a difference between asks and bids above the specified ratio, `max_spread_ratio` (defaults to `0.005`). - -Example: - -If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` and this pair will be filtered out. - -### Full example of Pairlist Handlers - -The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 priceunit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value. - -```json -"exchange": { - "pair_whitelist": [], - "pair_blacklist": ["BNB/BTC"] -}, -"pairlists": [ - { - "method": "VolumePairList", - "number_assets": 20, - "sort_key": "quoteVolume", - }, - {"method": "PrecisionFilter"}, - {"method": "PriceFilter", "low_price_ratio": 0.01}, - {"method": "SpreadFilter", "max_spread_ratio": 0.005}, - {"method": "ShuffleFilter", "seed": 42} - ], -``` +Obviously, if only one side is using limit orders, different pricing combinations can be used. +--8<-- "includes/pairlists.md" +--8<-- "includes/protections.md" ## Switch to Dry-run mode diff --git a/docs/data-analysis.md b/docs/data-analysis.md index fc4693b17..17da98935 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -1,12 +1,22 @@ # Analyzing bot data with Jupyter notebooks -You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/`. +You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/` after initializing the user directory with `freqtrade create-userdir --userdir user_data`. -## Pro tips +## Quick start with docker + +Freqtrade provides a docker-compose file which starts up a jupyter lab server. +You can run this server using the following command: `docker-compose -f docker/docker-compose-jupyter.yml up` + +This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`. +Please use the link that's printed in the console after startup for simplified login. + +For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analayis-using-docker-compose) section. + +### Pro tips * See [jupyter.org](https://jupyter.org/documentation) for usage instructions. * Don't forget to start a Jupyter notebook server from within your conda or venv environment or use [nb_conda_kernels](https://github.com/Anaconda-Platform/nb_conda_kernels)* -* Copy the example notebook before use so your changes don't get clobbered with the next freqtrade update. +* Copy the example notebook before use so your changes don't get overwritten with the next freqtrade update. ### Using virtual environment with system-wide Jupyter installation @@ -28,10 +38,8 @@ ipython kernel install --user --name=freqtrade !!! Note This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get jupyter notebooks up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community). - -## Fine print - -Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually. +!!! Warning + Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually. ## Recommended workflow diff --git a/docs/data-download.md b/docs/data-download.md index 903d62854..2d77a8a17 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -8,82 +8,121 @@ If no additional parameter is specified, freqtrade will download data for `"1m"` Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. +You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used. + !!! Tip "Tip: Updating existing data" If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. - Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. + Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. ### Usage ``` -usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] - [--pairs-file FILE] [--days INT] [--dl-trades] [--exchange EXCHANGE] - [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] - [--erase] [--data-format-ohlcv {json,jsongz}] [--data-format-trades {json,jsongz}] - -optional arguments: - -h, --help show this help message and exit - -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] - Show profits for only these pairs. Pairs are space-separated. - --pairs-file FILE File containing a list of pairs to download. - --days INT Download data for given number of days. - --dl-trades Download trades instead of OHLCV data. The bot will resample trades to the desired timeframe as specified as - --timeframes/-t. - --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. - -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] - Specify which tickers to download. Space-separated list. Default: `1m 5m`. - --erase Clean all existing data for the selected exchange/pairs/timeframes. - --data-format-ohlcv {json,jsongz} - Storage format for downloaded candle (OHLCV) data. (default: `json`). - --data-format-trades {json,jsongz} - Storage format for downloaded trades data. (default: `jsongz`). - -Common arguments: - -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. - -V, --version show program's version number and exit - -c PATH, --config PATH - Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` - to read config from stdin. - -d PATH, --datadir PATH - Path to directory with historical backtesting data. - --userdir PATH, --user-data-dir PATH - Path to userdata directory. -``` - -### Data format - -Freqtrade currently supports 2 dataformats, `json` (plain "text" json files) and `jsongz` (a gzipped version of json files). -By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. - -This can be changed via the `--data-format-ohlcv` and `--data-format-trades` parameters respectivly. - -If the default dataformat has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well. - -!!! Note - You can convert between data-formats using the [convert-data](#subcommand-convert-data) and [convert-trade-data](#subcommand-convert-trade-data) methods. - -#### Subcommand convert data - -``` -usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] - [-p PAIRS [PAIRS ...]] --format-from - {json,jsongz} --format-to {json,jsongz} - [--erase] - [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] +usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] [--pairs-file FILE] + [--days INT] [--timerange TIMERANGE] + [--dl-trades] [--exchange EXCHANGE] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] + [--erase] + [--data-format-ohlcv {json,jsongz,hdf5}] + [--data-format-trades {json,jsongz,hdf5}] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Show profits for only these pairs. Pairs are space- separated. - --format-from {json,jsongz} + --pairs-file FILE File containing a list of pairs to download. + --days INT Download data for given number of days. + --timerange TIMERANGE + Specify what timerange of data to use. + --dl-trades Download trades instead of OHLCV data. The bot will + resample trades to the desired timeframe as specified + as --timeframes/-t. + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no + config is provided. + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...] + Specify which tickers to download. Space-separated + list. Default: `1m 5m`. + --erase Clean all existing data for the selected + exchange/pairs/timeframes. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `json`). + --data-format-trades {json,jsongz,hdf5} + Storage format for downloaded trades data. (default: + `jsongz`). + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` + +!!! Note "Startup period" + `download-data` is a strategy-independent command. The idea is to download a big chunk of data once, and then iteratively increase the amount of data stored. + + For that reason, `download-data` does not care about the "startup-period" defined in a strategy. It's up to the user to download additional days if the backtest should start at a specific point in time (while respecting startup period). + +### Data format + +Freqtrade currently supports 3 data-formats for both OHLCV and trades data: + +* `json` (plain "text" json files) +* `jsongz` (a gzip-zipped version of json files) +* `hdf5` (a high performance datastore) + +By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. + +This can be changed via the `--data-format-ohlcv` and `--data-format-trades` command line arguments respectively. +To persist this change, you can should also add the following snippet to your configuration, so you don't have to insert the above arguments each time: + +``` jsonc + // ... + "dataformat_ohlcv": "hdf5", + "dataformat_trades": "hdf5", + // ... +``` + +If the default data-format has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well. + +!!! Note + You can convert between data-formats using the [convert-data](#sub-command-convert-data) and [convert-trade-data](#sub-command-convert-trade-data) methods. + +#### Sub-command convert data + +``` +usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] --format-from + {json,jsongz,hdf5} --format-to + {json,jsongz,hdf5} [--erase] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. + --format-from {json,jsongz,hdf5} Source format for data conversion. - --format-to {json,jsongz} + --format-to {json,jsongz,hdf5} Destination format for data conversion. --erase Clean all existing data for the selected exchange/pairs/timeframes. - -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...] Specify which tickers to download. Space-separated list. Default: `1m 5m`. @@ -94,9 +133,10 @@ Common arguments: details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -109,26 +149,26 @@ The following command will convert all candle (OHLCV) data available in `~/.freq It'll also remove original json data files (`--erase` parameter). ``` bash -freqtrade convert-data --format-from json --format-to jsongz --data-dir ~/.freqtrade/data/binance -t 5m 15m --erase +freqtrade convert-data --format-from json --format-to jsongz --datadir ~/.freqtrade/data/binance -t 5m 15m --erase ``` -#### Subcommand convert-trade data +#### Sub-command convert trade data ``` usage: freqtrade convert-trade-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] --format-from - {json,jsongz} --format-to {json,jsongz} - [--erase] + {json,jsongz,hdf5} --format-to + {json,jsongz,hdf5} [--erase] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Show profits for only these pairs. Pairs are space- separated. - --format-from {json,jsongz} + --format-from {json,jsongz,hdf5} Source format for data conversion. - --format-to {json,jsongz} + --format-to {json,jsongz,hdf5} Destination format for data conversion. --erase Clean all existing data for the selected exchange/pairs/timeframes. @@ -140,13 +180,15 @@ Common arguments: details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH Path to userdata directory. + ``` ##### Example converting trades @@ -155,7 +197,60 @@ The following command will convert all available trade-data in `~/.freqtrade/dat It'll also remove original jsongz data files (`--erase` parameter). ``` bash -freqtrade convert-trade-data --format-from jsongz --format-to json --data-dir ~/.freqtrade/data/kraken --erase +freqtrade convert-trade-data --format-from jsongz --format-to json --datadir ~/.freqtrade/data/kraken --erase +``` + +### Sub-command list-data + +You can get a list of downloaded data using the `list-data` sub-command. + +``` +usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] + [--userdir PATH] [--exchange EXCHANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] + [-p PAIRS [PAIRS ...]] + +optional arguments: + -h, --help show this help message and exit + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no + config is provided. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `json`). + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +``` + +#### Example list-data + +```bash +> freqtrade list-data --userdir ~/.freqtrade/user_data/ + +Found 33 pair / timeframe combinations. +pairs timeframe +---------- ----------------------------------------- +ADA/BTC 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d +ADA/ETH 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d +ETH/BTC 5m, 15m, 30m, 1h, 2h, 4h, 6h, 12h, 1d +ETH/USDT 5m, 15m, 30m, 1h, 2h, 4h ``` ### Pairs file @@ -197,15 +292,16 @@ This will download historical candle (OHLCV) data for all the currency pairs you ### Other Notes - To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. -- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust ratelimits etc.) +- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust rate limits etc.) - To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. - To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days). +- To download historical candle (OHLCV) data from a fixed starting point, use `--timerange 20200101-` - which will download all data from January 1st, 2020. Eventually set end dates are ignored. - Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. ### Trades (tick) data -By default, `download-data` subcommand downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API. +By default, `download-data` sub-command downloads Candles (OHLCV) data. Some exchanges also provide historic trade-data via their API. This data can be useful if you need many different timeframes, since it is only downloaded once, and then resampled locally to the desired timeframes. Since this data is large by default, the files use gzip by default. They are stored in your data-directory with the naming convention of `-trades.json.gz` (`ETH_BTC-trades.json.gz`). Incremental mode is also supported, as for historic OHLCV data, so downloading the data once per week with `--days 8` will create an incremental data-repository. diff --git a/docs/deprecated.md b/docs/deprecated.md index a7b57b10e..312f2c74f 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -9,21 +9,20 @@ and are no longer supported. Please avoid their usage in your configuration. ### the `--refresh-pairs-cached` command line option `--refresh-pairs-cached` in the context of backtesting, hyperopt and edge allows to refresh candle data for backtesting. -Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out -as a seperate freqtrade subcommand `freqtrade download-data`. +Since this leads to much confusion, and slows down backtesting (while not being part of backtesting) this has been singled out as a separate freqtrade sub-command `freqtrade download-data`. -This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9 (master branch). +This command line option was deprecated in 2019.7-dev (develop branch) and removed in 2019.9. ### The **--dynamic-whitelist** command line option This command line option was deprecated in 2018 and removed freqtrade 2019.6-dev (develop branch) -and in freqtrade 2019.7 (master branch). +and in freqtrade 2019.7. ### the `--live` command line option `--live` in the context of backtesting allowed to download the latest tick data for backtesting. Did only download the latest 500 candles, so was ineffective in getting good backtest data. -Removed in 2019-7-dev (develop branch) and in freqtrade 2019-8 (master branch) +Removed in 2019-7-dev (develop branch) and in freqtrade 2019.8. ### Allow running multiple pairlists in sequence @@ -31,6 +30,6 @@ The former `"pairlist"` section in the configuration has been removed, and is re The old section of configuration parameters (`"pairlist"`) has been deprecated in 2019.11 and has been removed in 2020.4. -### deprecation of bidVolume and askVolume from volumepairlist +### deprecation of bidVolume and askVolume from volume-pairlist -Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4. +Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4, and have been removed in 2020.9. diff --git a/docs/developer.md b/docs/developer.md index 036109d5b..dcbaa3ca9 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel in [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg) where you can ask questions. ## Documentation @@ -10,13 +10,35 @@ Documentation is available at [https://freqtrade.io](https://www.freqtrade.io/) Special fields for the documentation (like Note boxes, ...) can be found [here](https://squidfunk.github.io/mkdocs-material/extensions/admonition/). +To test the documentation locally use the following commands. + +``` bash +pip install -r docs/requirements-docs.txt +mkdocs serve +``` + +This will spin up a local server (usually on port 8000) so you can see if everything looks as you'd like it to. + ## Developer setup -To configure a development environment, best use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ". -Alternatively (if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`. +To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ". +Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`. This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`. +### Devcontainer setup + +The fastest and easiest way to get started is to use [VSCode](https://code.visualstudio.com/) with the Remote container extension. +This gives developers the ability to start the bot with all required dependencies *without* needing to install any freqtrade specific dependencies on your local machine. + +#### Devcontainer dependencies + +* [VSCode](https://code.visualstudio.com/) +* [docker](https://docs.docker.com/install/) +* [Remote container extension documentation](https://code.visualstudio.com/docs/remote) + +For more information about the [Remote container extension](https://code.visualstudio.com/docs/remote), best consult the documentation. + ### Tests New code should be covered by basic unittests. Depending on the complexity of the feature, Reviewers may request more in-depth unittests. @@ -41,53 +63,42 @@ def test_method_to_test(caplog): ``` -### Local docker usage +## ErrorHandling -The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine. +Freqtrade Exceptions all inherit from `FreqtradeException`. +This general class of error should however not be used directly. Instead, multiple specialized sub-Exceptions exist. -#### Install +Below is an outline of exception inheritance hierarchy: -* [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) -* [docker](https://docs.docker.com/install/) -* [docker-compose](https://docs.docker.com/compose/install/) - -#### Starting the bot -##### Use the develop dockerfile - -``` bash -rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml +``` ++ FreqtradeException +| ++---+ OperationalException +| ++---+ DependencyException +| | +| +---+ PricingError +| | +| +---+ ExchangeError +| | +| +---+ TemporaryError +| | +| +---+ DDosProtection +| | +| +---+ InvalidOrderException +| | +| +---+ RetryableOrderError +| | +| +---+ InsufficientFundsError +| ++---+ StrategyError ``` -#### Docker Compose +--- -##### Starting +## Plugins -``` bash -docker-compose up -``` - -![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png) - -##### Rebuilding - -``` bash -docker-compose build -``` - -##### Execing (effectively SSH into the container) - -The `exec` command requires that the container already be running, if you want to start it -that can be effected by `docker-compose up` or `docker-compose run freqtrade_develop` - -``` bash -docker-compose exec freqtrade_develop /bin/bash -``` - -![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png) - -## Modules - -### Dynamic Pairlist +### Pairlists You have a great idea for a new pair selection algorithm you would like to try out? Great. Hopefully you also want to contribute this back upstream. @@ -98,7 +109,7 @@ First of all, have a look at the [VolumePairList](https://github.com/freqtrade/f This is a simple Handler, which however serves as a good example on how to start developing. -Next, modify the classname of the Handler (ideally align this with the module filename). +Next, modify the class-name of the Handler (ideally align this with the module filename). The base-class provides an instance of the exchange (`self._exchange`) the pairlist manager (`self._pairlistmanager`), as well as the main configuration (`self._config`), the pairlist dedicated configuration (`self._pairlistconfig`) and the absolute position within the list of pairlists. @@ -110,6 +121,9 @@ The base-class provides an instance of the exchange (`self._exchange`) the pairl self._pairlist_pos = pairlist_pos ``` +!!! Tip + Don't forget to register your pairlist in `constants.py` under the variable `AVAILABLE_PAIRLISTS` - otherwise it will not be selectable. + Now, let's step through the methods which require actions: #### Pairlist configuration @@ -118,7 +132,7 @@ Configuration for the chain of Pairlist Handlers is done in the bot configuratio By convention, `"number_assets"` is used to specify the maximum number of pairs to keep in the pairlist. Please follow this to ensure a consistent user experience. -Additional parameters can be configured as needed. For instance, `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successfull and dynamic. +Additional parameters can be configured as needed. For instance, `VolumePairList` uses `"sort_key"` to specify the sorting value - however feel free to specify whatever is necessary for your great algorithm to be successful and dynamic. #### short_desc @@ -134,7 +148,7 @@ This is called with each iteration of the bot (only if the Pairlist Handler is a It must return the resulting pairlist (which may then be passed into the chain of Pairlist Handlers). -Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected. +Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filtering. Use this if you limit your result to a certain number of pairs - so the end-result is not shorter than expected. #### filter_pairlist @@ -142,13 +156,13 @@ This method is called for each Pairlist Handler in the chain by the pairlist man This is called with each iteration of the bot - so consider implementing caching for compute/network heavy calculations. -It get's passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`. +It gets passed a pairlist (which can be the result of previous pairlists) as well as `tickers`, a pre-fetched version of `get_tickers()`. The default implementation in the base class simply calls the `_validate_pair()` method for each pair in the pairlist, but you may override it. So you should either implement the `_validate_pair()` in your Pairlist Handler or override `filter_pairlist()` to do something else. If overridden, it must return the resulting pairlist (which may then be passed into the next Pairlist Handler in the chain). -Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the endresult is not shorter than expected. +Validations are optional, the parent class exposes a `_verify_blacklist(pairlist)` and `_whitelist_for_active_markets(pairlist)` to do default filters. Use this if you limit your result to a certain number of pairs - so the end result is not shorter than expected. In `VolumePairList`, this implements different methods of sorting, does early validation so only the expected number of pairs is returned. @@ -161,6 +175,66 @@ In `VolumePairList`, this implements different methods of sorting, does early va return pairs ``` +### Protections + +Best read the [Protection documentation](configuration.md#protections) to understand protections. +This Guide is directed towards Developers who want to develop a new protection. + +No protection should use datetime directly, but use the provided `date_now` variable for date calculations. This preserves the ability to backtest protections. + +!!! Tip "Writing a new Protection" + Best copy one of the existing Protections to have a good example. + Don't forget to register your protection in `constants.py` under the variable `AVAILABLE_PROTECTIONS` - otherwise it will not be selectable. + +#### Implementation of a new protection + +All Protection implementations must have `IProtection` as parent class. +For that reason, they must implement the following methods: + +* `short_desc()` +* `global_stop()` +* `stop_per_pair()`. + +`global_stop()` and `stop_per_pair()` must return a ProtectionReturn tuple, which consists of: + +* lock pair - boolean +* lock until - datetime - until when should the pair be locked (will be rounded up to the next new candle) +* reason - string, used for logging and storage in the database + +The `until` portion should be calculated using the provided `calculate_lock_end()` method. + +All Protections should use `"stop_duration"` / `"stop_duration_candles"` to define how long a a pair (or all pairs) should be locked. +The content of this is made available as `self._stop_duration` to the each Protection. + +If your protection requires a look-back period, please use `"lookback_period"` / `"lockback_period_candles"` to keep all protections aligned. + +#### Global vs. local stops + +Protections can have 2 different ways to stop trading for a limited : + +* Per pair (local) +* For all Pairs (globally) + +##### Protections - per pair + +Protections that implement the per pair approach must set `has_local_stop=True`. +The method `stop_per_pair()` will be called whenever a trade closed (sell order completed). + +##### Protections - global protection + +These Protections should do their evaluation across all pairs, and consequently will also lock all pairs from trading (called a global PairLock). +Global protection must set `has_global_stop=True` to be evaluated for global stops. +The method `global_stop()` will be called whenever a trade closed (sell order completed). + +##### Protections - calculating lock end time + +Protections should calculate the lock end time based on the last trade it considers. +This avoids re-locking should the lookback-period be longer than the actual lock period. + +The `IProtection` parent class provides a helper method for this in `calculate_lock_end()`. + +--- + ## Implement a new Exchange (WIP) !!! Note @@ -172,7 +246,7 @@ Most exchanges supported by CCXT should work out of the box. Check if the new exchange supports Stoploss on Exchange orders through their API. -Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselfs. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects. +Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need to implement the exchange-specific parameters ourselves. Best look at `binance.py` for an example implementation of this. You'll need to dig through the documentation of the Exchange's API on how exactly this can be done. [CCXT Issues](https://github.com/ccxt/ccxt/issues) may also provide great help, since others may have implemented something similar for their projects. ### Incomplete candles @@ -222,13 +296,14 @@ jupyter nbconvert --ClearOutputPreprocessor.enabled=True --to markdown freqtrade This documents some decisions taken for the CI Pipeline. * CI runs on all OS variants, Linux (ubuntu), macOS and Windows. -* Docker images are build for the branches `master` and `develop`. -* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:master_pi` and `develop_pi`. +* Docker images are build for the branches `stable` and `develop`. +* Docker images containing Plot dependencies are also available as `stable_plot` and `develop_plot`. +* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:stable_pi` and `develop_pi`. * Docker images contain a file, `/freqtrade/freqtrade_commit` containing the commit this image is based of. * Full docker image rebuilds are run once a week via schedule. * Deployments run on ubuntu. * ta-lib binaries are contained in the build_helpers directory to avoid fails related to external unavailability. -* All tests must pass for a PR to be merged to `master` or `develop`. +* All tests must pass for a PR to be merged to `stable` or `develop`. ## Creating a release @@ -245,21 +320,22 @@ git checkout -b new_release Determine if crucial bugfixes have been made between this commit and the current state, and eventually cherry-pick these. +* Merge the release branch (stable) into this branch. * Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi. * Commit this part -* push that branch to the remote and create a PR against the master branch +* push that branch to the remote and create a PR against the stable branch ### Create changelog from git commits !!! Note - Make sure that the master branch is uptodate! + Make sure that the `stable` branch is up-to-date! ``` bash # Needs to be done before merging / pulling that branch. -git log --oneline --no-decorate --no-merges master..new_release +git log --oneline --no-decorate --no-merges stable..new_release ``` -To keep the release-log short, best wrap the full git changelog into a collapsible details secction. +To keep the release-log short, best wrap the full git changelog into a collapsible details section. ```markdown
@@ -272,17 +348,20 @@ To keep the release-log short, best wrap the full git changelog into a collapsib ### Create github release / tag -Once the PR against master is merged (best right after merging): +Once the PR against stable is merged (best right after merging): * Use the button "Draft a new release" in the Github UI (subsection releases). * Use the version-number specified as tag. -* Use "master" as reference (this step comes after the above PR is merged). +* Use "stable" as reference (this step comes after the above PR is merged). * Use the above changelog as release comment (as codeblock) ## Releases ### pypi +!!! Note + This process is now automated as part of Github Actions. + To create a pypi release, please run the following commands: Additional requirement: `wheel`, `twine` (for uploading), account on pypi with proper permissions. diff --git a/docs/docker.md b/docs/docker.md index 92478088a..f4699cf4c 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -1,145 +1,7 @@ -# Using Freqtrade with Docker - -## Install Docker - -Start by downloading and installing Docker CE for your platform: - -* [Mac](https://docs.docker.com/docker-for-mac/install/) -* [Windows](https://docs.docker.com/docker-for-windows/install/) -* [Linux](https://docs.docker.com/install/) - -Optionally, [docker-compose](https://docs.docker.com/compose/install/) should be installed and available to follow the [docker quick start guide](#docker-quick-start). - -Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. - -## Freqtrade with docker-compose - -Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage. - -!!! Note - The following section assumes that docker and docker-compose is installed and available to the logged in user. - -!!! Note - All below comands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file. - -!!! Note "Docker on Raspberry" - If you're running freqtrade on a Raspberry PI, you must change the image from `freqtradeorg/freqtrade:master` to `freqtradeorg/freqtrade:master_pi` or `freqtradeorg/freqtrade:develop_pi`, otherwise the image will not work. - -### Docker quick start - -Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory. - -``` bash -mkdir ft_userdata -cd ft_userdata/ -# Download the docker-compose file from the repository -curl https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docker-compose.yml -o docker-compose.yml - -# Pull the freqtrade image -docker-compose pull - -# Create user directory structure -docker-compose run --rm freqtrade create-userdir --userdir user_data - -# Create configuration - Requires answering interactive questions -docker-compose run --rm freqtrade new-config --config user_data/config.json -``` - -The above snippet creates a new directory called "ft_userdata", downloads the latest compose file and pulls the freqtrade image. -The last 2 steps in the snippet create the directory with user-data, as well as (interactively) the default configuration based on your selections. - -!!! Note - You can edit the configuration at any time, which is available as `user_data/config.json` (within the directory `ft_userdata`) when using the above configuration. - -#### Adding your strategy - -The configuration is now available as `user_data/config.json`. -You should now copy your strategy to `user_data/strategies/` - and add the Strategy class name to the `docker-compose.yml` file, replacing `SampleStrategy`. If you wish to run the bot with the SampleStrategy, just leave it as it is. - -!!! Warning - The `SampleStrategy` is there for your reference and give you ideas for your own strategy. - Please always backtest the strategy and use dry-run for some time before risking real money! - -Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above). - -``` bash -docker-compose up -d -``` - -#### Docker-compose logs - -Logs will be written to `user_data/logs/freqtrade.log`. -Alternatively, you can check the latest logs using `docker-compose logs -f`. - -#### Database - -The database will be in the user_data directory as well, and will be called `user_data/tradesv3.sqlite`. - -#### Updating freqtrade with docker-compose - -To update freqtrade when using docker-compose is as simple as running the following 2 commands: - -``` bash -# Download the latest image -docker-compose pull -# Restart the image -docker-compose up -d -``` - -This will first pull the latest image, and will then restart the container with the just pulled version. - -!!! Note - You should always check the changelog for breaking changes / manual interventions required and make sure the bot starts correctly after the update. - -#### Going from here - -Advanced users may edit the docker-compose file further to include all possible options or arguments. - -All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. - -!!! Note "`docker-compose run --rm`" - Including `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). - -##### Example: Download data with docker-compose - -Download backtesting data for 5 days for the pair ETH/BTC and 1h timeframe from Binance. The data will be stored in the directory `user_data/data/` on the host. - -``` bash -docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h -``` - -Head over to the [Data Downloading Documentation](data-download.md) for more details on downloading data. - -##### Example: Backtest with docker-compose - -Run backtesting in docker-containers for SampleStrategy and specified timerange of historical data, on 5m timeframe: - -``` bash -docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m -``` - -Head over to the [Backtesting Documentation](backtesting.md) to learn more. - -#### Additional dependencies with docker-compose - -If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host. -For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) for an example). - -You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions. - -``` yaml - image: freqtrade_custom - build: - context: . - dockerfile: "./Dockerfile." -``` - -You can then run `docker-compose build` to build the docker image, and run it using the commands described above. - ## Freqtrade with docker without docker-compose !!! Warning - The below documentation is provided for completeness and assumes that you are somewhat familiar with running docker containers. If you're just starting out with docker, we recommend to follow the [Freqtrade with docker-compose](#freqtrade-with-docker-compose) instructions. + The below documentation is provided for completeness and assumes that you are familiar with running docker containers. If you're just starting out with Docker, we recommend to follow the [Quickstart](docker.md) instructions. ### Download the official Freqtrade docker image @@ -148,9 +10,9 @@ Pull the image from docker hub. Branches / tags available can be checked out on [Dockerhub tags page](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). ```bash -docker pull freqtradeorg/freqtrade:develop +docker pull freqtradeorg/freqtrade:stable # Optionally tag the repository so the run-commands remain shorter -docker tag freqtradeorg/freqtrade:develop freqtrade +docker tag freqtradeorg/freqtrade:stable freqtrade ``` To update the image, simply run the above commands again and restart your running container. @@ -158,7 +20,7 @@ To update the image, simply run the above commands again and restart your runnin Should you require additional libraries, please [build the image yourself](#build-your-own-docker-image). !!! Note "Docker image update frequency" - The official docker images with tags `master`, `develop` and `latest` are automatically rebuild once a week to keep the base image uptodate. + The official docker images with tags `stable`, `develop` and `latest` are automatically rebuild once a week to keep the base image up-to-date. In addition to that, every merge to `develop` will trigger a rebuild for `develop` and `latest`. ### Prepare the configuration files @@ -190,39 +52,38 @@ cp -n config.json.example config.json #### Create your database file -Production +=== "Dry-Run" + ``` bash + touch tradesv3.dryrun.sqlite + ``` -```bash -touch tradesv3.sqlite -```` +=== "Production" + ``` bash + touch tradesv3.sqlite + ``` -Dry-Run -```bash -touch tradesv3.dryrun.sqlite -``` - -!!! Note - Make sure to use the path to this file when starting the bot in docker. +!!! Warning "Database File Path" + Make sure to use the path to the correct database file when starting the bot in Docker. ### Build your own Docker image Best start by pulling the official docker image from dockerhub as explained [here](#download-the-official-docker-image) to speed up building. -To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image. +To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image. ```bash -docker build -t freqtrade -f Dockerfile.technical . +docker build -t freqtrade -f docker/Dockerfile.technical . ``` -If you are developing using Docker, use `Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies: +If you are developing using Docker, use `docker/Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies: ```bash -docker build -f Dockerfile.develop -t freqtrade-dev . +docker build -f docker/Dockerfile.develop -t freqtrade-dev . ``` -!!! Note - For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see the "5. Run a restartable docker image" section) to keep it between updates. +!!! Warning "Include your config file manually" + For security reasons, your configuration file will not be included in the image, you will need to bind mount it. It is also advised to bind mount an SQLite database file (see [5. Run a restartable docker image](#run-a-restartable-docker-image)") to keep it between updates. #### Verify the Docker image @@ -243,37 +104,36 @@ docker run --rm -v `pwd`/config.json:/freqtrade/config.json -it freqtrade ``` !!! Warning - In this example, the database will be created inside the docker instance and will be lost when you will refresh your image. + In this example, the database will be created inside the docker instance and will be lost when you refresh your image. #### Adjust timezone By default, the container will use UTC timezone. -Should you find this irritating please add the following to your docker commands: +If you would like to change the timezone use the following commands: -##### Linux +=== "Linux" + ``` bash + -v /etc/timezone:/etc/timezone:ro -``` bash --v /etc/timezone:/etc/timezone:ro + # Complete command: + docker run --rm -v /etc/timezone:/etc/timezone:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade + ``` -# Complete command: -docker run --rm -v /etc/timezone:/etc/timezone:ro -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` +=== "MacOS" + ```bash + docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade + ``` -##### MacOS - -There is known issue in OSX Docker versions after 17.09.1, whereby `/etc/localtime` cannot be shared causing Docker to not start. A work-around for this is to start with the following cmd. - -```bash -docker run --rm -e TZ=`ls -la /etc/localtime | cut -d/ -f8-9` -v `pwd`/config.json:/freqtrade/config.json -it freqtrade -``` - -More information on this docker issue and work-around can be read [here](https://github.com/docker/for-mac/issues/2396). +!!! Note "MacOS Issues" + The OSX Docker versions after 17.09.1 have a known issue whereby `/etc/localtime` cannot be shared causing Docker to not start.
+ A work-around for this is to start with the MacOS command above + More information on this docker issue and work-around can be read [here](https://github.com/docker/for-mac/issues/2396). ### Run a restartable docker image To run a restartable instance in the background (feel free to place your configuration and database files wherever it feels comfortable on your filesystem). -#### Move your config file and database +#### 1. Move your config file and database The following will assume that you place your configuration / database files to `~/.freqtrade`, which is a hidden directory in your home directory. Feel free to use a different directory and replace the directory in the upcomming commands. @@ -283,7 +143,7 @@ mv config.json ~/.freqtrade mv tradesv3.sqlite ~/.freqtrade ``` -#### Run the docker image +#### 2. Run the docker image ```bash docker run -d \ diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md new file mode 100644 index 000000000..48ee34954 --- /dev/null +++ b/docs/docker_quickstart.md @@ -0,0 +1,191 @@ +# Using Freqtrade with Docker + +## Install Docker + +Start by downloading and installing Docker CE for your platform: + +* [Mac](https://docs.docker.com/docker-for-mac/install/) +* [Windows](https://docs.docker.com/docker-for-windows/install/) +* [Linux](https://docs.docker.com/install/) + +Optionally, [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the [docker quick start guide](#docker-quick-start). + +Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. + +## Freqtrade with docker-compose + +Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage. + +!!! Note + - The following section assumes that `docker` and `docker-compose` are installed and available to the logged in user. + - All below commands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file. + +### Docker quick start + +Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory. + +=== "PC/MAC/Linux" + ``` bash + mkdir ft_userdata + cd ft_userdata/ + # Download the docker-compose file from the repository + curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml + + # Pull the freqtrade image + docker-compose pull + + # Create user directory structure + docker-compose run --rm freqtrade create-userdir --userdir user_data + + # Create configuration - Requires answering interactive questions + docker-compose run --rm freqtrade new-config --config user_data/config.json + ``` + +=== "RaspberryPi" + ``` bash + mkdir ft_userdata + cd ft_userdata/ + # Download the docker-compose file from the repository + curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml + + # Pull the freqtrade image + docker-compose pull + + # Create user directory structure + docker-compose run --rm freqtrade create-userdir --userdir user_data + + # Create configuration - Requires answering interactive questions + docker-compose run --rm freqtrade new-config --config user_data/config.json + ``` + + !!! Note "Change your docker Image" + You have to change the docker image in the docker-compose file for your Raspberry build to work properly. + ``` yml + image: freqtradeorg/freqtrade:stable_pi + # image: freqtradeorg/freqtrade:develop_pi + ``` + +The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image. +The last 2 steps in the snippet create the directory with `user_data`, as well as (interactively) the default configuration based on your selections. + +!!! Question "How to edit the bot configuration?" + You can edit the configuration at any time, which is available as `user_data/config.json` (within the directory `ft_userdata`) when using the above configuration. + + You can also change the both Strategy and commands by editing the `docker-compose.yml` file. + +#### Adding a custom strategy + +1. The configuration is now available as `user_data/config.json` +2. Copy a custom strategy to the directory `user_data/strategies/` +3. add the Strategy' class name to the `docker-compose.yml` file + +The `SampleStrategy` is run by default. + +!!! Warning "`SampleStrategy` is just a demo!" + The `SampleStrategy` is there for your reference and give you ideas for your own strategy. + Please always backtest the strategy and use dry-run for some time before risking real money! + +Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above). + +``` bash +docker-compose up -d +``` + +#### Docker-compose logs + +Logs will be located at: `user_data/logs/freqtrade.log`. +You can check the latest log with the command `docker-compose logs -f`. + +#### Database + +The database will be at: `user_data/tradesv3.sqlite` + +#### Updating freqtrade with docker-compose + +To update freqtrade when using `docker-compose` is as simple as running the following 2 commands: + +``` bash +# Download the latest image +docker-compose pull +# Restart the image +docker-compose up -d +``` + +This will first pull the latest image, and will then restart the container with the just pulled version. + +!!! Warning "Check the Changelog" + You should always check the changelog for breaking changes / manual interventions required and make sure the bot starts correctly after the update. + +### Editing the docker-compose file + +Advanced users may edit the docker-compose file further to include all possible options or arguments. + +All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. + +!!! Note "`docker-compose run --rm`" + Including `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). + +#### Example: Download data with docker-compose + +Download backtesting data for 5 days for the pair ETH/BTC and 1h timeframe from Binance. The data will be stored in the directory `user_data/data/` on the host. + +``` bash +docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h +``` + +Head over to the [Data Downloading Documentation](data-download.md) for more details on downloading data. + +#### Example: Backtest with docker-compose + +Run backtesting in docker-containers for SampleStrategy and specified timerange of historical data, on 5m timeframe: + +``` bash +docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m +``` + +Head over to the [Backtesting Documentation](backtesting.md) to learn more. + +### Additional dependencies with docker-compose + +If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host. +For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) for an example). + +You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions. + +``` yaml + image: freqtrade_custom + build: + context: . + dockerfile: "./Dockerfile." +``` + +You can then run `docker-compose build` to build the docker image, and run it using the commands described above. + +## Plotting with docker-compose + +Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file. +You can then use these commands as follows: + +``` bash +docker-compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p BTC/ETH --timerange=20180801-20180805 +``` + +The output will be stored in the `user_data/plot` directory, and can be opened with any modern browser. + +## Data analayis using docker compose + +Freqtrade provides a docker-compose file which starts up a jupyter lab server. +You can run this server using the following command: + +``` bash +docker-compose --rm -f docker/docker-compose-jupyter.yml up +``` + +This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`. +Please use the link that's printed in the console after startup for simplified login. + +Since part of this image is built on your machine, it is recommended to rebuild the image from time to time to keep freqtrade (and dependencies) uptodate. + +``` bash +docker-compose -f docker/docker-compose-jupyter.yml build --no-cache +``` diff --git a/docs/edge.md b/docs/edge.md index bdcf6cde9..fd6d2cf7d 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,95 +1,160 @@ # Edge positioning -This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. +The `Edge Positioning` module uses probability to calculate your win rate and risk reward ration. It will use these statistics to control your strategy trade entry points, position side and, stoploss. !!! Warning - Edge positioning is not compatible with dynamic (volume-based) whitelist. + `Edge positioning` is not compatible with dynamic (volume-based) whitelist. !!! Note - Edge does not consider anything else than buy/sell/stoploss signals. So trailing stoploss, ROI, and everything else are ignored in its calculation. + `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file. + `Edge Positioning` improves the performance of some trading strategies and *decreases* the performance of others. ## Introduction -Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose. +Trading strategies are not perfect. They are frameworks that are susceptible to the market and its indicators. Because the market is not at all predictable, sometimes a strategy will win and sometimes the same strategy will lose. -But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: you give me 10$. Is it an interesting game? No, it's quite boring, isn't it? +To obtain an edge in the market, a strategy has to make more money than it loses. Making money in trading is not only about *how often* the strategy makes or loses money. -But let's say the probability that we have heads is 80% (because our coin has the displaced distribution of mass or other defect), and the probability that we have tails is 20%. Now it is becoming interesting... +!!! tip "It doesn't matter how often, but how much!" + A bad strategy might make 1 penny in *ten* transactions but lose 1 dollar in *one* transaction. If one only checks the number of winning trades, it would be misleading to think that the strategy is actually making a profit. -That means 10$ X 80% versus 10$ X 20%. 8$ versus 2$. That means over time you will win 8$ risking only 2$ on each toss of coin. +The Edge Positioning module seeks to improve a strategy's winning probability and the money that the strategy will make *on the long run*. -Let's complicate it more: you win 80% of the time but only 2$, I win 20% of the time but 8$. The calculation is: 80% X 2$ versus 20% X 8$. It is becoming boring again because overtime you win $1.6$ (80% X 2$) and me $1.6 (20% X 8$) too. +We raise the following question[^1]: -The question is: How do you calculate that? How do you know if you wanna play? +!!! Question "Which trade is a better option?" + a) A trade with 80% of chance of losing 100\$ and 20% chance of winning 200\$
+ b) A trade with 100% of chance of losing 30\$ -The answer comes to two factors: +???+ Info "Answer" + The expected value of *a)* is smaller than the expected value of *b)*.
+ Hence, *b*) represents a smaller loss in the long run.
+ However, the answer is: *it depends* -- Win Rate -- Risk Reward Ratio +Another way to look at it is to ask a similar question: -### Win Rate +!!! Question "Which trade is a better option?" + a) A trade with 80% of chance of winning 100\$ and 20% chance of losing 200\$
+ b) A trade with 100% of chance of winning 30\$ -Win Rate (*W*) is is the mean over some amount of trades (*N*) what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only if you won or not). +Edge positioning tries to answer the hard questions about risk/reward and position size automatically, seeking to minimizes the chances of losing of a given strategy. -``` -W = (Number of winning trades) / (Total number of trades) = (Number of winning trades) / N -``` +### Trading, winning and losing -Complementary Loss Rate (*L*) is defined as +Let's call $o$ the return of a single transaction $o$ where $o \in \mathbb{R}$. The collection $O = \{o_1, o_2, ..., o_N\}$ is the set of all returns of transactions made during a trading session. We say that $N$ is the cardinality of $O$, or, in lay terms, it is the number of transactions made in a trading session. -``` -L = (Number of losing trades) / (Total number of trades) = (Number of losing trades) / N -``` +!!! Example + In a session where a strategy made three transactions we can say that $O = \{3.5, -1, 15\}$. That means that $N = 3$ and $o_1 = 3.5$, $o_2 = -1$, $o_3 = 15$. -or, which is the same, as +A winning trade is a trade where a strategy *made* money. Making money means that the strategy closed the position in a value that returned a profit, after all deducted fees. Formally, a winning trade will have a return $o_i > 0$. Similarly, a losing trade will have a return $o_j \leq 0$. With that, we can discover the set of all winning trades, $T_{win}$, as follows: -``` -L = 1 – W -``` +$$ T_{win} = \{ o \in O | o > 0 \} $$ + +Similarly, we can discover the set of losing trades $T_{lose}$ as follows: + +$$ T_{lose} = \{o \in O | o \leq 0\} $$ + +!!! Example + In a section where a strategy made three transactions $O = \{3.5, -1, 15, 0\}$:
+ $T_{win} = \{3.5, 15\}$
+ $T_{lose} = \{-1, 0\}$
+ +### Win Rate and Lose Rate + +The win rate $W$ is the proportion of winning trades with respect to all the trades made by a strategy. We use the following function to compute the win rate: + +$$W = \frac{|T_{win}|}{N}$$ + +Where $W$ is the win rate, $N$ is the number of trades and, $T_{win}$ is the set of all trades where the strategy made money. + +Similarly, we can compute the rate of losing trades: + +$$ + L = \frac{|T_{lose}|}{N} +$$ + +Where $L$ is the lose rate, $N$ is the amount of trades made and, $T_{lose}$ is the set of all trades where the strategy lost money. Note that the above formula is the same as calculating $L = 1 – W$ or $W = 1 – L$ ### Risk Reward Ratio -Risk Reward Ratio (*R*) is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: +Risk Reward Ratio ($R$) is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose. Formally: -``` -R = Profit / Loss -``` +$$ R = \frac{\text{potential_profit}}{\text{potential_loss}} $$ -Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: +???+ Example "Worked example of $R$ calculation" + Let's say that you think that the price of *stonecoin* today is 10.0\$. You believe that, because they will start mining stonecoin, it will go up to 15.0\$ tomorrow. There is the risk that the stone is too hard, and the GPUs can't mine it, so the price might go to 0\$ tomorrow. You are planning to invest 100\$, which will give you 10 shares (100 / 10). -``` -Average profit = (Sum of profits) / (Number of winning trades) + Your potential profit is calculated as: -Average loss = (Sum of losses) / (Number of losing trades) + $\begin{aligned} + \text{potential_profit} &= (\text{potential_price} - \text{entry_price}) * \frac{\text{investment}}{\text{entry_price}} \\ + &= (15 - 10) * (100 / 10) \\ + &= 50 + \end{aligned}$ -R = (Average profit) / (Average loss) -``` + Since the price might go to 0\$, the 100\$ dollars invested could turn into 0. + + We do however use a stoploss of 15% - so in the worst case, we'll sell 15% below entry price (or at 8.5$\). + + $\begin{aligned} + \text{potential_loss} &= (\text{entry_price} - \text{stoploss}) * \frac{\text{investment}}{\text{entry_price}} \\ + &= (10 - 8.5) * (100 / 10)\\ + &= 15 + \end{aligned}$ + + We can compute the Risk Reward Ratio as follows: + + $\begin{aligned} + R &= \frac{\text{potential_profit}}{\text{potential_loss}}\\ + &= \frac{50}{15}\\ + &= 3.33 + \end{aligned}$
+ What it effectively means is that the strategy have the potential to make 3.33\$ for each 1\$ invested. + +On a long horizon, that is, on many trades, we can calculate the risk reward by dividing the strategy' average profit on winning trades by the strategy' average loss on losing trades. We can calculate the average profit, $\mu_{win}$, as follows: + +$$ \text{average_profit} = \mu_{win} = \frac{\text{sum_of_profits}}{\text{count_winning_trades}} = \frac{\sum^{o \in T_{win}} o}{|T_{win}|} $$ + +Similarly, we can calculate the average loss, $\mu_{lose}$, as follows: + +$$ \text{average_loss} = \mu_{lose} = \frac{\text{sum_of_losses}}{\text{count_losing_trades}} = \frac{\sum^{o \in T_{lose}} o}{|T_{lose}|} $$ + +Finally, we can calculate the Risk Reward ratio, $R$, as follows: + +$$ R = \frac{\text{average_profit}}{\text{average_loss}} = \frac{\mu_{win}}{\mu_{lose}}\\ $$ + + +???+ Example "Worked example of $R$ calculation using mean profit/loss" + Let's say the strategy that we are using makes an average win $\mu_{win} = 2.06$ and an average loss $\mu_{loss} = 4.11$.
+ We calculate the risk reward ratio as follows:
+ $R = \frac{\mu_{win}}{\mu_{loss}} = \frac{2.06}{4.11} = 0.5012...$ + ### Expectancy -At this point we can combine *W* and *R* to create an expectancy ratio. This is a simple process of multiplying the risk reward ratio by the percentage of winning trades and subtracting the percentage of losing trades, which is calculated as follows: +By combining the Win Rate $W$ and and the Risk Reward ratio $R$ to create an expectancy ratio $E$. A expectance ratio is the expected return of the investment made in a trade. We can compute the value of $E$ as follows: -``` -Expectancy Ratio = (Risk Reward Ratio X Win Rate) – Loss Rate = (R X W) – L -``` +$$E = R * W - L$$ -So lets say your Win rate is 28% and your Risk Reward Ratio is 5: +!!! Example "Calculating $E$" + Let's say that a strategy has a win rate $W = 0.28$ and a risk reward ratio $R = 5$. What this means is that the strategy is expected to make 5 times the investment around on 28% of the trades it makes. Working out the example:
+ $E = R * W - L = 5 * 0.28 - 0.72 = 0.68$ +
-``` -Expectancy = (5 X 0.28) – 0.72 = 0.68 -``` +The expectancy worked out in the example above means that, on average, this strategy' trades will return 1.68 times the size of its losses. Said another way, the strategy makes 1.68\$ for every 1\$ it loses, on average. -Superficially, this means that on average you expect this strategy’s trades to return 1.68 times the size of your loses. Said another way, you can expect to win $1.68 for every $1 you lose. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. +This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. You can also use this value to evaluate the effectiveness of modifications to this system. -**NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data, there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. +!!! Note + It's important to keep in mind that Edge is testing your expectancy using historical data, there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. ## How does it work? -If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over *N* trades for each stoploss. Here is an example: +Edge combines dynamic stoploss, dynamic positions, and whitelist generation into one isolated module which is then applied to the trading strategy. If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over *N* trades for each stoploss. Here is an example: | Pair | Stoploss | Win Rate | Risk Reward Ratio | Expectancy | |----------|:-------------:|-------------:|------------------:|-----------:| @@ -98,13 +163,13 @@ If enabled in config, Edge will go through historical data with a range of stopl | XZC/ETH | -0.03 | 0.52 |1.359670 | 0.228 | | XZC/ETH | -0.04 | 0.51 |1.234539 | 0.117 | -The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. In the above example stoploss at 3% leads to the maximum expectancy according to historical data. +The goal here is to find the best stoploss for the strategy in order to have the maximum expectancy. In the above example stoploss at $3%$ leads to the maximum expectancy according to historical data. Edge module then forces stoploss value it evaluated to your strategy dynamically. ### Position size -Edge also dictates the stake amount for each trade to the bot according to the following factors: +Edge dictates the amount at stake for each trade to the bot according to the following factors: - Allowed capital at risk - Stoploss @@ -115,9 +180,9 @@ Allowed capital at risk is calculated as follows: Allowed capital at risk = (Capital available_percentage) X (Allowed risk per trade) ``` -Stoploss is calculated as described above against historical data. +Stoploss is calculated as described above with respect to historical data. -Your position size then will be: +The position size is calculated as follows: ``` Position size = (Allowed capital at risk) / Stoploss @@ -125,19 +190,23 @@ Position size = (Allowed capital at risk) / Stoploss Example: -Let's say the stake currency is ETH and you have 10 ETH on the exchange, your capital available percentage is 50% and you would allow 1% of risk for each trade. thus your available capital for trading is **10 x 0.5 = 5 ETH** and allowed capital at risk would be **5 x 0.01 = 0.05 ETH**. +Let's say the stake currency is **ETH** and there is $10$ **ETH** on the wallet. The capital available percentage is $50%$ and the allowed risk per trade is $1\%$. Thus, the available capital for trading is $10 * 0.5 = 5$ **ETH** and the allowed capital at risk would be $5 * 0.01 = 0.05$ **ETH**. -Let's assume Edge has calculated that for **XLM/ETH** market your stoploss should be at 2%. So your position size will be **0.05 / 0.02 = 2.5 ETH**. +- **Trade 1:** The strategy detects a new buy signal in the **XLM/ETH** market. `Edge Positioning` calculates a stoploss of $2\%$ and a position of $0.05 / 0.02 = 2.5$ **ETH**. The bot takes a position of $2.5$ **ETH** in the **XLM/ETH** market. -Bot takes a position of 2.5 ETH on XLM/ETH (call it trade 1). Up next, you receive another buy signal while trade 1 is still open. This time on **BTC/ETH** market. Edge calculated stoploss for this market at 4%. So your position size would be 0.05 / 0.04 = 1.25 ETH (call it trade 2). +- **Trade 2:** The strategy detects a buy signal on the **BTC/ETH** market while **Trade 1** is still open. `Edge Positioning` calculates the stoploss of $4\%$ on this market. Thus, **Trade 2** position size is $0.05 / 0.04 = 1.25$ **ETH**. -Note that available capital for trading didn’t change for trade 2 even if you had already trade 1. The available capital doesn’t mean the free amount on your wallet. +!!! Tip "Available Capital $\neq$ Available in wallet" + The available capital for trading didn't change in **Trade 2** even with **Trade 1** still open. The available capital **is not** the free amount in the wallet. -Now you have two trades open. The bot receives yet another buy signal for another market: **ADA/ETH**. This time the stoploss is calculated at 1%. So your position size is **0.05 / 0.01 = 5 ETH**. But there are already 3.75 ETH blocked in two previous trades. So the position size for this third trade would be **5 – 3.75 = 1.25 ETH**. +- **Trade 3:** The strategy detects a buy signal in the **ADA/ETH** market. `Edge Positioning` calculates a stoploss of $1\%$ and a position of $0.05 / 0.01 = 5$ **ETH**. Since **Trade 1** has $2.5$ **ETH** blocked and **Trade 2** has $1.25$ **ETH** blocked, there is only $5 - 1.25 - 2.5 = 1.25$ **ETH** available. Hence, the position size of **Trade 3** is $1.25$ **ETH**. -Available capital doesn’t change before a position is sold. Let’s assume that trade 1 receives a sell signal and it is sold with a profit of 1 ETH. Your total capital on exchange would be 11 ETH and the available capital for trading becomes 5.5 ETH. +!!! Tip "Available Capital Updates" + The available capital does not change before a position is sold. After a trade is closed the Available Capital goes up if the trade was profitable or goes down if the trade was a loss. -So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be **0.055 / 0.02 = 2.75 ETH**. +- The strategy detects a sell signal in the **XLM/ETH** market. The bot exits **Trade 1** for a profit of $1$ **ETH**. The total capital in the wallet becomes $11$ **ETH** and the available capital for trading becomes $5.5$ **ETH**. + +- **Trade 4** The strategy detects a new buy signal int the **XLM/ETH** market. `Edge Positioning` calculates the stoploss of $2%$, and the position size of $0.055 / 0.02 = 2.75$ **ETH**. ## Configurations @@ -153,9 +222,9 @@ Edge module has following configuration options: | `stoploss_range_max` | Maximum stoploss.
*Defaults to `-0.10`.*
**Datatype:** Float | `stoploss_range_step` | As an example if this is set to -0.01 then Edge will test the strategy for `[-0.01, -0,02, -0,03 ..., -0.09, -0.10]` ranges.
**Note** than having a smaller step means having a bigger range which could lead to slow calculation.
If you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10.
*Defaults to `-0.001`.*
**Datatype:** Float | `minimum_winrate` | It filters out pairs which don't have at least minimum_winrate.
This comes handy if you want to be conservative and don't comprise win rate in favour of risk reward ratio.
*Defaults to `0.60`.*
**Datatype:** Float -| `minimum_expectancy` | It filters out pairs which have the expectancy lower than this number.
Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return.
*Defaults to `0.20`.*
**Datatype:** Float +| `minimum_expectancy` | It filters out pairs which have the expectancy lower than this number.
Having an expectancy of 0.20 means if you put 10\$ on a trade you expect a 12\$ return.
*Defaults to `0.20`.*
**Datatype:** Float | `min_trade_number` | When calculating *W*, *R* and *E* (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable.
Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
*Defaults to `10` (it is highly recommended not to decrease this number).*
**Datatype:** Integer -| `max_trade_duration_minute` | Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
**NOTICE:** While configuring this value, you should take into consideration your timeframe (ticker interval). As an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. Default value is set assuming your strategy interval is relatively small (1m or 5m, etc.).
*Defaults to `1440` (one day).*
**Datatype:** Integer +| `max_trade_duration_minute` | Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
**NOTICE:** While configuring this value, you should take into consideration your timeframe. As an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. Default value is set assuming your strategy interval is relatively small (1m or 5m, etc.).
*Defaults to `1440` (one day).*
**Datatype:** Integer | `remove_pumps` | Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
*Defaults to `false`.*
**Datatype:** Boolean ## Running Edge independently @@ -168,23 +237,29 @@ freqtrade edge An example of its output: -| pair | stoploss | win rate | risk reward ratio | required risk reward | expectancy | total number of trades | average duration (min) | -|:----------|-----------:|-----------:|--------------------:|-----------------------:|-------------:|-------------------------:|-------------------------:| -| AGI/BTC | -0.02 | 0.64 | 5.86 | 0.56 | 3.41 | 14 | 54 | -| NXS/BTC | -0.03 | 0.64 | 2.99 | 0.57 | 1.54 | 11 | 26 | -| LEND/BTC | -0.02 | 0.82 | 2.05 | 0.22 | 1.50 | 11 | 36 | -| VIA/BTC | -0.01 | 0.55 | 3.01 | 0.83 | 1.19 | 11 | 48 | -| MTH/BTC | -0.09 | 0.56 | 2.82 | 0.80 | 1.12 | 18 | 52 | -| ARDR/BTC | -0.04 | 0.42 | 3.14 | 1.40 | 0.73 | 12 | 42 | -| BCPT/BTC | -0.01 | 0.71 | 1.34 | 0.40 | 0.67 | 14 | 30 | -| WINGS/BTC | -0.02 | 0.56 | 1.97 | 0.80 | 0.65 | 27 | 42 | -| VIBE/BTC | -0.02 | 0.83 | 0.91 | 0.20 | 0.59 | 12 | 35 | -| MCO/BTC | -0.02 | 0.79 | 0.97 | 0.27 | 0.55 | 14 | 31 | -| GNT/BTC | -0.02 | 0.50 | 2.06 | 1.00 | 0.53 | 18 | 24 | -| HOT/BTC | -0.01 | 0.17 | 7.72 | 4.81 | 0.50 | 209 | 7 | -| SNM/BTC | -0.03 | 0.71 | 1.06 | 0.42 | 0.45 | 17 | 38 | -| APPC/BTC | -0.02 | 0.44 | 2.28 | 1.27 | 0.44 | 25 | 43 | -| NEBL/BTC | -0.03 | 0.63 | 1.29 | 0.58 | 0.44 | 19 | 59 | +| **pair** | **stoploss** | **win rate** | **risk reward ratio** | **required risk reward** | **expectancy** | **total number of trades** | **average duration (min)** | +|:----------|-----------:|-----------:|--------------------:|-----------------------:|-------------:|-----------------:|---------------:| +| **AGI/BTC** | -0.02 | 0.64 | 5.86 | 0.56 | 3.41 | 14 | 54 | +| **NXS/BTC** | -0.03 | 0.64 | 2.99 | 0.57 | 1.54 | 11 | 26 | +| **LEND/BTC** | -0.02 | 0.82 | 2.05 | 0.22 | 1.50 | 11 | 36 | +| **VIA/BTC** | -0.01 | 0.55 | 3.01 | 0.83 | 1.19 | 11 | 48 | +| **MTH/BTC** | -0.09 | 0.56 | 2.82 | 0.80 | 1.12 | 18 | 52 | +| **ARDR/BTC** | -0.04 | 0.42 | 3.14 | 1.40 | 0.73 | 12 | 42 | +| **BCPT/BTC** | -0.01 | 0.71 | 1.34 | 0.40 | 0.67 | 14 | 30 | +| **WINGS/BTC** | -0.02 | 0.56 | 1.97 | 0.80 | 0.65 | 27 | 42 | +| **VIBE/BTC** | -0.02 | 0.83 | 0.91 | 0.20 | 0.59 | 12 | 35 | +| **MCO/BTC** | -0.02 | 0.79 | 0.97 | 0.27 | 0.55 | 14 | 31 | +| **GNT/BTC** | -0.02 | 0.50 | 2.06 | 1.00 | 0.53 | 18 | 24 | +| **HOT/BTC** | -0.01 | 0.17 | 7.72 | 4.81 | 0.50 | 209 | 7 | +| **SNM/BTC** | -0.03 | 0.71 | 1.06 | 0.42 | 0.45 | 17 | 38 | +| **APPC/BTC** | -0.02 | 0.44 | 2.28 | 1.27 | 0.44 | 25 | 43 | +| **NEBL/BTC** | -0.03 | 0.63 | 1.29 | 0.58 | 0.44 | 19 | 59 | + +Edge produced the above table by comparing `calculate_since_number_of_days` to `minimum_expectancy` to find `min_trade_number` historical information based on the config file. The timerange Edge uses for its comparisons can be further limited by using the `--timerange` switch. + +In live and dry-run modes, after the `process_throttle_secs` has passed, Edge will again process `calculate_since_number_of_days` against `minimum_expectancy` to find `min_trade_number`. If no `min_trade_number` is found, the bot will return "whitelist empty". Depending on the trade strategy being deployed, "whitelist empty" may be return much of the time - or *all* of the time. The use of Edge may also cause trading to occur in bursts, though this is rare. + +If you encounter "whitelist empty" a lot, condsider tuning `calculate_since_number_of_days`, `minimum_expectancy` and `min_trade_number` to align to the trading frequency of your strategy. ### Update cached pairs with the latest data @@ -211,3 +286,6 @@ The full timerange specification: * Use tickframes since 2018/01/31: `--timerange=20180131-` * Use tickframes since 2018/01/31 till 2018/03/01 : `--timerange=20180131-20180301` * Use tickframes between POSIX timestamps 1527595200 1527618600: `--timerange=1527595200-1527618600` + + +[^1]: Question extracted from MIT Opencourseware S096 - Mathematics with applications in Finance: https://ocw.mit.edu/courses/mathematics/18-s096-topics-in-mathematics-with-applications-in-finance-fall-2013/ diff --git a/docs/exchanges.md b/docs/exchanges.md index aef356ea9..d877e6da2 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -23,7 +23,8 @@ Binance has been split into 3, and users must use the correct ccxt exchange ID f ## Kraken !!! Tip "Stoploss on Exchange" - Kraken supports `stoploss_on_exchange` and uses stop-loss-market orders. It provides great advantages, so we recommend to benefit from it, however since the resulting order is a stoploss-market order, sell-rates are not guaranteed, which makes this feature less secure than on other exchanges. This limitation is based on kraken's policy [source](https://blog.kraken.com/post/1234/announcement-delisting-pairs-and-temporary-suspension-of-advanced-order-types/) and [source2](https://blog.kraken.com/post/1494/kraken-enables-advanced-orders-and-adds-10-currency-pairs/) - which has stoploss-limit orders disabled. + Kraken supports `stoploss_on_exchange` and can use both stop-loss-market and stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. + You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use. ### Historic Kraken data @@ -73,6 +74,10 @@ print(res) ## FTX +!!! Tip "Stoploss on Exchange" + FTX supports `stoploss_on_exchange` and can use both stop-loss-market and stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. + You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type of stoploss shall be used. + ### Using subaccounts To use subaccounts with FTX, you need to edit the configuration and add the following: @@ -94,10 +99,10 @@ To use subaccounts with FTX, you need to edit the configuration and add the foll Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys. - ## Random notes for other exchanges * The Ocean (exchange id: `theocean`) exchange uses Web3 functionality and requires `web3` python package to be installed: + ```shell $ pip3 install web3 ``` diff --git a/docs/faq.md b/docs/faq.md index 8e8a1bf35..b424cd31d 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,25 +1,31 @@ # Freqtrade FAQ +## Beginner Tips & Tricks + +* When you work with your strategy & hyperopt file you should use a proper code editor like VSCode or PyCharm. A good code editor will provide syntax highlighting as well as line numbers, making it easy to find syntax errors (most likely pointed out by Freqtrade during startup). + ## Freqtrade common issues ### The bot does not start -Running the bot with `freqtrade trade --config config.json` does show the output `freqtrade: command not found`. +Running the bot with `freqtrade trade --config config.json` shows the output `freqtrade: command not found`. -This could have the following reasons: +This could be caused by the following reasons: -* The virtual environment is not active - * run `source .env/bin/activate` to activate the virtual environment +* The virtual environment is not active. + * Run `source .env/bin/activate` to activate the virtual environment. * The installation did not work correctly. * Please check the [Installation documentation](installation.md). -### I have waited 5 minutes, why hasn't the bot made any trades yet?! +### I have waited 5 minutes, why hasn't the bot made any trades yet? -Depending on the buy strategy, the amount of whitelisted coins, the -situation of the market etc, it can take up to hours to find good entry +* Depending on the buy strategy, the amount of whitelisted coins, the +situation of the market etc, it can take up to hours to find a good entry position for a trade. Be patient! -### I have made 12 trades already, why is my total profit negative?! +* It may be because of a configuration error. It's best to check the logs, they usually tell you if the bot is simply not getting buy signals (only heartbeat messages), or if there is something wrong (errors / exceptions in the log). + +### I have made 12 trades already, why is my total profit negative? I understand your disappointment but unfortunately 12 trades is just not enough to say anything. If you run backtesting, you can see that our @@ -30,11 +36,9 @@ of course constantly aim to improve the bot but it will _always_ be a gamble, which should leave you with modest wins on monthly basis but you can't say much from few trades. -### I’d like to change the stake amount. Can I just stop the bot with /stop and then change the config.json and run it again? +### I’d like to make changes to the config. Can I do that without having to kill the bot? -Not quite. Trades are persisted to a database but the configuration is -currently only read when the bot is killed and restarted. `/stop` more -like pauses. You can stop your bot, adjust settings and start it again. +Yes. You can edit your config, use the `/stop` command in Telegram, followed by `/reload_config` and the bot will run with the new config. ### I want to improve the bot with a new strategy @@ -43,7 +47,21 @@ the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-c ### Is there a setting to only SELL the coins being held and not perform anymore BUYS? -You can use the `/forcesell all` command from Telegram. +You can use the `/stopbuy` command in Telegram to prevent future buys, followed by `/forcesell all` (sell all open trades). + +### I want to run multiple bots on the same machine + +Please look at the [advanced setup documentation Page](advanced-setup.md#running-multiple-instances-of-freqtrade). + +### I'm getting "Missing data fillup" messages in the log + +This message is just a warning that the latest candles had missing candles in them. +Depending on the exchange, this can indicate that the pair didn't have a trade for the timeframe you are using - and the exchange does only return candles with volume. +On low volume pairs, this is a rather common occurrence. + +If this happens for all pairs in the pairlist, this might indicate a recent exchange downtime. Please check your exchange's public channels for details. + +Irrespectively of the reason, Freqtrade will fill up these candles with "empty" candles, where open, high, low and close are set to the previous candle close - and volume is empty. In a chart, this will look like a `_` - and is aligned with how exchanges usually represent 0 volume candles. ### I'm getting the "RESTRICTED_MARKET" message in the log @@ -53,7 +71,7 @@ Read [the Bittrex section about restricted markets](exchanges.md#restricted-mark ### I'm getting the "Exchange Bittrex does not support market orders." message and cannot run my strategy -As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Probably your strategy was written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex). +As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex). To fix it for Bittrex, redefine order types in the strategy to use "limit" instead of "market": @@ -65,11 +83,11 @@ To fix it for Bittrex, redefine order types in the strategy to use "limit" inste } ``` -Same fix should be done in the configuration file, if order types are defined in your custom config rather than in the strategy. +The same fix should be applied in the configuration file, if order types are defined in your custom config rather than in the strategy. ### How do I search the bot logs for something? -By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility subcommands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout. +By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility sub-commands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout. * In unix shells, this normally can be done as simple as: ```shell @@ -94,7 +112,7 @@ and then grep it as: ```shell $ cat /path/to/mylogfile.log | grep 'something' ``` -or even on the fly, as the bot works and the logfile grows: +or even on the fly, as the bot works and the log file grows: ```shell $ tail -f /path/to/mylogfile.log | grep 'something' ``` @@ -107,43 +125,46 @@ On Windows, the `--logfile` option is also supported by Freqtrade and you can us ## Hyperopt module -### How many epoch do I need to get a good Hyperopt result? +### How many epochs do I need to get a good Hyperopt result? Per default Hyperopt called without the `-e`/`--epochs` command line option will only -run 100 epochs, means 100 evals of your triggers, guards, ... Too few +run 100 epochs, means 100 evaluations of your triggers, guards, ... Too few to find a great result (unless if you are very lucky), so you probably have to run it for 10.000 or more. But it will take an eternity to compute. -We recommend you to run it at least 10.000 epochs: +Since hyperopt uses Bayesian search, running for too many epochs may not produce greater results. + +It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epochs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going. ```bash -freqtrade hyperopt -e 10000 +freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy SampleStrategy -e 1000 ``` -or if you want intermediate result to see +### Why does it take a long time to run hyperopt? -```bash -for i in {1..100}; do freqtrade hyperopt -e 100; done -``` +* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. -### Why it is so long to run hyperopt? +* If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers: -Finding a great Hyperopt results takes time. +This answer was written during the release 0.15.1, when we had: -If you wonder why it takes a while to find great hyperopt results - -This answer was written during the under the release 0.15.1, when we had: - -- 8 triggers -- 9 guards: let's say we evaluate even 10 values from each -- 1 stoploss calculation: let's say we want 10 values from that too to be evaluated +* 8 triggers +* 9 guards: let's say we evaluate even 10 values from each +* 1 stoploss calculation: let's say we want 10 values from that too to be evaluated The following calculation is still very rough and not very precise but it will give the idea. With only these triggers and guards there is -already 8\*10^9\*10 evaluations. A roughly total of 80 billion evals. -Did you run 100 000 evals? Congrats, you've done roughly 1 / 100 000 th -of the search space. +already 8\*10^9\*10 evaluations. A roughly total of 80 billion evaluations. +Did you run 100 000 evaluations? Congrats, you've done roughly 1 / 100 000 th +of the search space, assuming that the bot never tests the same parameters more than once. + +* The time it takes to run 1000 hyperopt epochs depends on things like: The available cpu, hard-disk, ram, timeframe, timerange, indicator settings, indicator count, amount of coins that hyperopt test strategies on and the resulting trade count - which can be 650 trades in a year or 10.0000 trades depending if the strategy aims for big profits by trading rarely or for many low profit trades. + +Example: 4% profit 650 times vs 0,3% profit a trade 10.000 times in a year. If we assume you set the --timerange to 365 days. + +Example: +`freqtrade --config config.json --strategy SampleStrategy --hyperopt SampleHyperopt -e 1000 --timerange 20190601-20200601` ## Edge module @@ -151,7 +172,7 @@ of the search space. The Edge module is mostly a result of brainstorming of [@mishaker](https://github.com/mishaker) and [@creslinux](https://github.com/creslinux) freqtrade team members. -You can find further info on expectancy, winrate, risk management and position size in the following sources: +You can find further info on expectancy, win rate, risk management and position size in the following sources: - https://www.tradeciety.com/ultimate-math-guide-for-traders/ - http://www.vantharp.com/tharp-concepts/expectancy.asp diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 8efc51a39..f88d9cd4f 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -37,12 +37,20 @@ pip install -r requirements-hyperopt.txt Before we start digging into Hyperopt, we recommend you to take a look at the sample hyperopt file located in [user_data/hyperopts/](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt.py). -Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar and a lot of code can be copied across from the strategy. +Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar. -The simplest way to get started is to use `freqtrade new-hyperopt --hyperopt AwesomeHyperopt`. -This will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`. +!!! Tip "About this page" + For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class. -### Checklist on all tasks / possibilities in hyperopt +The simplest way to get started is to use the following, command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`. + +``` bash +freqtrade new-hyperopt --hyperopt AwesomeHyperopt +``` + +### Hyperopt checklist + +Checklist on all tasks / possibilities in hyperopt Depending on the space you want to optimize, only some of the below are required: @@ -54,17 +62,15 @@ Depending on the space you want to optimize, only some of the below are required !!! Note `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. -Optional - can also be loaded from a strategy: +Optional in hyperopt - can also be loaded from a strategy (recommended): -* copy `populate_indicators` from your strategy - otherwise default-strategy will be used -* copy `populate_buy_trend` from your strategy - otherwise default-strategy will be used -* copy `populate_sell_trend` from your strategy - otherwise default-strategy will be used - -!!! Note - Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. +* `populate_indicators` - fallback to create indicators +* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy +* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy !!! Note You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods. + Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. Rarely you may also need to override: @@ -80,36 +86,41 @@ Rarely you may also need to override: # Have a working strategy at hand. freqtrade new-hyperopt --hyperopt EmptyHyperopt - freqtrade hyperopt --hyperopt EmptyHyperopt --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100 + freqtrade hyperopt --hyperopt EmptyHyperopt --hyperopt-loss SharpeHyperOptLossDaily --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100 ``` -### 1. Install a Custom Hyperopt File +### Create a Custom Hyperopt File -Put your hyperopt file into the directory `user_data/hyperopts`. +Let assume you want a hyperopt file `AwesomeHyperopt.py`: -Let assume you want a hyperopt file `awesome_hyperopt.py`: -Copy the file `user_data/hyperopts/sample_hyperopt.py` into `user_data/hyperopts/awesome_hyperopt.py` +``` bash +freqtrade new-hyperopt --hyperopt AwesomeHyperopt +``` -### 2. Configure your Guards and Triggers +This command will create a new hyperopt file from a template, allowing you to get started quickly. + +### Configure your Guards and Triggers There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: * Inside `indicator_space()` - the parameters hyperopt shall be optimizing. -* Inside `populate_buy_trend()` - applying the parameters. +* Within `buy_strategy_generator()` - populate the nested `populate_buy_trend()` to apply the parameters. There you have two different types of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. 2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower Bollinger band". -Hyperoptimization will, for each eval round, pick one trigger and possibly -multiple guards. The constructed strategy will be something like -"*buy exactly when close price touches lower Bollinger band, BUT only if +!!! Hint "Guards and Triggers" + Technically, there is no difference between Guards and Triggers. + However, this guide will make this distinction to make it clear that signals should not be "sticking". + Sticking signals are signals that are active for multiple candles. This can lead into buying a signal late (right before the signal disappears - which means that the chance of success is a lot lower than right at the beginning). + +Hyper-optimization will, for each epoch round, pick one trigger and possibly +multiple guards. The constructed strategy will be something like "*buy exactly when close price touches lower Bollinger band, BUT only if ADX > 10*". -If you have updated the buy strategy, i.e. changed the contents of -`populate_buy_trend()` method, you have to update the `guards` and -`triggers` your hyperopt must use correspondingly. +If you have updated the buy strategy, i.e. changed the contents of `populate_buy_trend()` method, you have to update the `guards` and `triggers` your hyperopt must use correspondingly. #### Sell optimization @@ -117,16 +128,16 @@ Similar to the buy-signal above, sell-signals can also be optimized. Place the corresponding settings into the following methods * Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. -* Inside `populate_sell_trend()` - applying the parameters. +* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters. The configuration and rules are the same than for buy signals. To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. #### Using timeframe as a part of the Strategy -The Strategy class exposes the timeframe (ticker interval) value as the `self.ticker_interval` attribute. -The same value is available as class-attribute `HyperoptName.ticker_interval`. -In the case of the linked sample-value this would be `SampleHyperOpt.ticker_interval`. +The Strategy class exposes the timeframe value as the `self.timeframe` attribute. +The same value is available as class-attribute `HyperoptName.timeframe`. +In the case of the linked sample-value this would be `AwesomeHyperopt.timeframe`. ## Solving a Mystery @@ -154,7 +165,7 @@ We will start by defining a search space: Above definition says: I have five parameters I want you to randomly combine to find the best combination. Two of them are integer values (`adx-value` -and `rsi-value`) and I want you test in the range of values 20 to 40. +and `rsi-value`) and I want you test in the range of values 20 to 40. Then we have three category variables. First two are either `True` or `False`. We use these to either enable or disable the ADX and RSI guards. The last one we call `trigger` and use it to decide which buy trigger we want to use. @@ -162,6 +173,11 @@ one we call `trigger` and use it to decide which buy trigger we want to use. So let's write the buy strategy using these values: ```python + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by Hyperopt. + """ def populate_buy_trend(dataframe: DataFrame) -> DataFrame: conditions = [] # GUARDS AND TRENDS @@ -192,27 +208,25 @@ So let's write the buy strategy using these values: return populate_buy_trend ``` -Hyperopting will now call this `populate_buy_trend` as many times you ask it (`epochs`) -with different value combinations. It will then use the given historical data and make -buys based on the buy signals generated with the above function and based on the results -it will end with telling you which parameter combination produced the best profits. +Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations. +It will use the given historical data and make buys based on the buy signals generated with the above function. +Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)). -The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. -When you want to test an indicator that isn't used by the bot currently, remember to -add it to the `populate_indicators()` method in your custom hyperopt file. +!!! Note + The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. + When you want to test an indicator that isn't used by the bot currently, remember to + add it to the `populate_indicators()` method in your strategy or hyperopt file. ## Loss-functions Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results. -By default, Freqtrade uses a loss function, which has been with freqtrade since the beginning and optimizes mostly for short trade duration and avoiding losses. - -A different loss function can be specified by using the `--hyperopt-loss ` argument. +A loss function must be specified via the `--hyperopt-loss ` argument (or optionally via the configuration under the `"hyperopt_loss"` key). This class should be in its own file within the `user_data/hyperopts/` directory. Currently, the following loss functions are builtin: -* `DefaultHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) +* `ShortTradeDurHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) - Mostly for short trade duration and avoiding losses. * `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) * `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on trade returns relative to standard deviation) * `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on **daily** trade returns relative to standard deviation) @@ -229,21 +243,20 @@ Because hyperopt tries a lot of combinations to find the best parameters it will We strongly recommend to use `screen` or `tmux` to prevent any connection loss. ```bash -freqtrade hyperopt --config config.json --hyperopt -e 5000 --spaces all +freqtrade hyperopt --config config.json --hyperopt --hyperopt-loss --strategy -e 500 --spaces all ``` Use `` as the name of the custom hyperopt used. -The `-e` option will set how many evaluations hyperopt will do. We recommend -running at least several thousand evaluations. +The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs. +Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results. The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below. !!! Note - By default, hyperopt will erase previous results and start from scratch. Continuation can be archived by using `--continue`. - -!!! Warning - When switching parameters or changing configuration options, make sure to not use the argument `--continue` so temporary results can be removed. + Hyperopt will store hyperopt results with the timestamp of the hyperopt start time. + Reading commands (`hyperopt-list`, `hyperopt-show`) can use `--hyperopt-filename ` to read and display older hyperopt results. + You can find a list of filenames with `ls -l user_data/hyperopt_results/`. ### Execute Hyperopt with different historical data source @@ -251,13 +264,13 @@ If you would like to hyperopt parameters using an alternate historical data set you have on-disk, use the `--datadir PATH` option. By default, hyperopt uses data from directory `user_data/data`. -### Running Hyperopt with Smaller Testset +### Running Hyperopt with a smaller test-set -Use the `--timerange` argument to change how much of the testset you want to use. +Use the `--timerange` argument to change how much of the test-set you want to use. For example, to use one month of data, pass the following parameter to the hyperopt call: ```bash -freqtrade hyperopt --timerange 20180401-20180501 +freqtrade hyperopt --hyperopt --strategy --timerange 20180401-20180501 ``` ### Running Hyperopt using methods from a strategy @@ -265,16 +278,15 @@ freqtrade hyperopt --timerange 20180401-20180501 Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided. ```bash -freqtrade hyperopt --strategy SampleStrategy --customhyperopt SampleHyperopt +freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy ``` ### Running Hyperopt with Smaller Search Space Use the `--spaces` option to limit the search space used by hyperopt. -Letting Hyperopt optimize everything is a huuuuge search space. Often it -might make more sense to start by just searching for initial buy algorithm. -Or maybe you just want to optimize your stoploss or roi table for that awesome -new buy strategy you have. +Letting Hyperopt optimize everything is a huuuuge search space. +Often it might make more sense to start by just searching for initial buy algorithm. +Or maybe you just want to optimize your stoploss or roi table for that awesome new buy strategy you have. Legal values are: @@ -318,7 +330,7 @@ The initial state for generation of these random values (random state) is contro If you have not set this value explicitly in the command line options, Hyperopt seeds the random state with some random value for you. The random state value for each Hyperopt run is shown in the log, so you can copy and paste it into the `--random-state` command line option to repeat the set of the initial random epochs used. -If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyperoptimization results with same random state value used. +If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyper-optimization results with same random state value used. ## Understand the Hyperopt Result @@ -370,6 +382,9 @@ By default, hyperopt prints colorized results -- epochs with positive profit are You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option. +!!! Note "Windows and color output" + Windows does not support color-output natively, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL. + ### Understand Hyperopt ROI results If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table: @@ -403,7 +418,7 @@ As stated in the comment, you can also use it as the value of the `minimal_roi` #### Default ROI Search Space -If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the ticker_interval used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 5 digits after the decimal point): +If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the timeframe used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 5 digits after the decimal point): | # step | 1m | | 5m | | 1h | | 1d | | | ------ | ------ | ----------------- | -------- | ----------- | ---------- | ----------------- | ------------ | ----------------- | @@ -412,11 +427,13 @@ If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace f | 3 | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 | | 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 | -These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe (ticker interval) used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used. +These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used. If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default. -Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). A sample for these methods can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). + +A sample for these methods can be found in [sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). ### Understand Hyperopt Stoploss results @@ -438,7 +455,7 @@ Stoploss: -0.27996 In order to use this best stoploss value found by Hyperopt in backtesting and for live trades/dry-run, copy-paste it as the value of the `stoploss` attribute of your custom strategy: -``` +``` python # Optimal stoploss designed for the strategy # This attribute will be overridden if the config file contains "stoploss" stoploss = -0.27996 @@ -472,7 +489,7 @@ Trailing stop: In order to use these best trailing stop parameters found by Hyperopt in backtesting and for live trades/dry-run, copy-paste them as the values of the corresponding attributes of your custom strategy: -``` +``` python # Trailing stop # These attributes will be overridden if the config file contains corresponding values. trailing_stop = True @@ -491,15 +508,14 @@ Override the `trailing_space()` method and define the desired range in it if you ## Show details of Hyperopt results -After you run Hyperopt for the desired amount of epochs, you can later list all results for analysis, select only best or profitable once, and show the details for any of the epochs previously evaluated. This can be done with the `hyperopt-list` and `hyperopt-show` subcommands. The usage of these subcommands is described in the [Utils](utils.md#list-hyperopt-results) chapter. +After you run Hyperopt for the desired amount of epochs, you can later list all results for analysis, select only best or profitable once, and show the details for any of the epochs previously evaluated. This can be done with the `hyperopt-list` and `hyperopt-show` sub-commands. The usage of these sub-commands is described in the [Utils](utils.md#list-hyperopt-results) chapter. ## Validate backtesting results Once the optimized strategy has been implemented into your strategy, you should backtest this strategy to make sure everything is working as expected. -To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same set of arguments `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. +To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. -## Next Step - -Now you have a perfect bot and want to control it from Telegram. Your -next step is to learn the [Telegram usage](telegram-usage.md). +Should results don't match, please double-check to make sure you transferred all conditions correctly. +Pay special care to the stoploss (and trailing stoploss) parameters, as these are often set in configuration files, which override changes to the strategy. +You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss` or `trailing_stop`). diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md new file mode 100644 index 000000000..732dfa5bb --- /dev/null +++ b/docs/includes/pairlists.md @@ -0,0 +1,186 @@ +## Pairlists and Pairlist Handlers + +Pairlist Handlers define the list of pairs (pairlist) that the bot should trade. They are configured in the `pairlists` section of the configuration settings. + +In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler). + +Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist. + +If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler. + +Inactive markets are always removed from the resulting pairlist. Explicitly blacklisted pairs (those in the `pair_blacklist` configuration setting) are also always removed from the resulting pairlist. + +### Available Pairlist Handlers + +* [`StaticPairList`](#static-pair-list) (default, if not configured differently) +* [`VolumePairList`](#volume-pair-list) +* [`AgeFilter`](#agefilter) +* [`PerformanceFilter`](#performancefilter) +* [`PrecisionFilter`](#precisionfilter) +* [`PriceFilter`](#pricefilter) +* [`ShuffleFilter`](#shufflefilter) +* [`SpreadFilter`](#spreadfilter) +* [`RangeStabilityFilter`](#rangestabilityfilter) + +!!! Tip "Testing pairlists" + Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly. + +#### Static Pair List + +By default, the `StaticPairList` method is used, which uses a statically defined pair whitelist from the configuration. + +It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklist`. + +```json +"pairlists": [ + {"method": "StaticPairList"} + ], +``` + +By default, only currently enabled pairs are allowed. +To skip pair validation against active markets, set `"allow_inactive": true` within the `StaticPairList` configuration. +This can be useful for backtesting expired pairs (like quarterly spot-markets). +This option must be configured along with `exchange.skip_pair_validation` in the exchange configuration. + +#### Volume Pair List + +`VolumePairList` employs sorting/filtering of pairs by their trading volume. It selects `number_assets` top pairs with sorting based on the `sort_key` (which can only be `quoteVolume`). + +When used in the chain of Pairlist Handlers in a non-leading position (after StaticPairList and other Pairlist Filters), `VolumePairList` considers outputs of previous Pairlist Handlers, adding its sorting/selection of the pairs by the trading volume. + +When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top assets from all available markets (with matching stake-currency) on the exchange. + +The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). + +`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library: + +* The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours. + +```json +"pairlists": [{ + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "refresh_period": 1800 +}], +``` + +!!! Note + `VolumePairList` does not support backtesting mode. + +#### AgeFilter + +Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`). + +When pairs are first listed on an exchange they can suffer huge price drops and volatility +in the first few days while the pair goes through its price-discovery period. Bots can often +be caught out buying before the pair has finished dropping in price. + +This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days. + +#### PerformanceFilter + +Sorts pairs by past trade performance, as follows: +1. Positive performance. +2. No closed trades yet. +3. Negative performance. + +Trade count is used as a tie breaker. + +!!! Note + `PerformanceFilter` does not support backtesting mode. + +#### PrecisionFilter + +Filters low-value coins which would not allow setting stoplosses. + +#### PriceFilter + +The `PriceFilter` allows filtering of pairs by price. Currently the following price filters are supported: + +* `min_price` +* `max_price` +* `low_price_ratio` + +The `min_price` setting removes pairs where the price is below the specified price. This is useful if you wish to avoid trading very low-priced pairs. +This option is disabled by default, and will only apply if set to > 0. + +The `max_price` setting removes pairs where the price is above the specified price. This is useful if you wish to trade only low-priced pairs. +This option is disabled by default, and will only apply if set to > 0. + +The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio. +This option is disabled by default, and will only apply if set to > 0. + +For `PriceFiler` at least one of its `min_price`, `max_price` or `low_price_ratio` settings must be applied. + +Calculation example: + +Min price precision for SHITCOIN/BTC is 8 decimals. If its price is 0.00000011 - one price step above would be 0.00000012, which is ~9% higher than the previous price value. You may filter out this pair by using PriceFilter with `low_price_ratio` set to 0.09 (9%) or with `min_price` set to 0.00000011, correspondingly. + +!!! Warning "Low priced pairs" + Low priced pairs with high "1 pip movements" are dangerous since they are often illiquid and it may also be impossible to place the desired stoploss, which can often result in high losses since price needs to be rounded to the next tradable price - so instead of having a stoploss of -5%, you could end up with a stoploss of -9% simply due to price rounding. + +#### ShuffleFilter + +Shuffles (randomizes) pairs in the pairlist. It can be used for preventing the bot from trading some of the pairs more frequently then others when you want all pairs be treated with the same priority. + +!!! Tip + You may set the `seed` value for this Pairlist to obtain reproducible results, which can be useful for repeated backtesting sessions. If `seed` is not set, the pairs are shuffled in the non-repeatable random order. + +#### SpreadFilter + +Removes pairs that have a difference between asks and bids above the specified ratio, `max_spread_ratio` (defaults to `0.005`). + +Example: + +If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027, the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` and this pair will be filtered out. + +#### RangeStabilityFilter + +Removes pairs where the difference between lowest low and highest high over `lookback_days` days is below `min_rate_of_change`. Since this is a filter that requires additional data, the results are cached for `refresh_period`. + +In the below example: +If the trading range over the last 10 days is <1%, remove the pair from the whitelist. + +```json +"pairlists": [ + { + "method": "RangeStabilityFilter", + "lookback_days": 10, + "min_rate_of_change": 0.01, + "refresh_period": 1440 + } +] +``` + +!!! Tip + This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit. + +### Full example of Pairlist Handlers + +The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value. + +```json +"exchange": { + "pair_whitelist": [], + "pair_blacklist": ["BNB/BTC"] +}, +"pairlists": [ + { + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + }, + {"method": "AgeFilter", "min_days_listed": 10}, + {"method": "PrecisionFilter"}, + {"method": "PriceFilter", "low_price_ratio": 0.01}, + {"method": "SpreadFilter", "max_spread_ratio": 0.005}, + { + "method": "RangeStabilityFilter", + "lookback_days": 10, + "min_rate_of_change": 0.01, + "refresh_period": 1440 + }, + {"method": "ShuffleFilter", "seed": 42} + ], +``` diff --git a/docs/includes/protections.md b/docs/includes/protections.md new file mode 100644 index 000000000..87db17fd8 --- /dev/null +++ b/docs/includes/protections.md @@ -0,0 +1,169 @@ +## Protections + +!!! Warning "Beta feature" + This feature is still in it's testing phase. Should you notice something you think is wrong please let us know via Discord, Slack or via Github Issue. + +Protections will protect your strategy from unexpected events and market conditions by temporarily stop trading for either one pair, or for all pairs. +All protection end times are rounded up to the next candle to avoid sudden, unexpected intra-candle buys. + +!!! Note + Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy to improve performance. + +!!! Tip + Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). + +!!! Note "Backtesting" + Protections are supported by backtesting and hyperopt, but must be explicitly enabled by using the `--enable-protections` flag. + +### Available Protections + +* [`StoplossGuard`](#stoploss-guard) Stop trading if a certain amount of stoploss occurred within a certain time window. +* [`MaxDrawdown`](#maxdrawdown) Stop trading if max-drawdown is reached. +* [`LowProfitPairs`](#low-profit-pairs) Lock pairs with low profits +* [`CooldownPeriod`](#cooldown-period) Don't enter a trade right after selling a trade. + +### Common settings to all Protections + +| Parameter| Description | +|------------|-------------| +| `method` | Protection name to use.
**Datatype:** String, selected from [available Protections](#available-protections) +| `stop_duration_candles` | For how many candles should the lock be set?
**Datatype:** Positive integer (in candles) +| `stop_duration` | how many minutes should protections be locked.
Cannot be used together with `stop_duration_candles`.
**Datatype:** Float (in minutes) +| `lookback_period_candles` | Only trades that completed within the last `lookback_period_candles` candles will be considered. This setting may be ignored by some Protections.
**Datatype:** Positive integer (in candles). +| `lookback_period` | Only trades that completed after `current_time - lookback_period` will be considered.
Cannot be used together with `lookback_period_candles`.
This setting may be ignored by some Protections.
**Datatype:** Float (in minutes) +| `trade_limit` | Number of trades required at minimum (not used by all Protections).
**Datatype:** Positive integer + +!!! Note "Durations" + Durations (`stop_duration*` and `lookback_period*` can be defined in either minutes or candles). + For more flexibility when testing different timeframes, all below examples will use the "candle" definition. + +#### Stoploss Guard + +`StoplossGuard` selects all trades within `lookback_period`, and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. +This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. + +The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. + +```json +"protections": [ + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 4, + "only_per_pair": false + } +], +``` + +!!! Note + `StoplossGuard` considers all trades with the results `"stop_loss"` and `"trailing_stop_loss"` if the resulting profit was negative. + `trade_limit` and `lookback_period` will need to be tuned for your strategy. + +#### MaxDrawdown + +`MaxDrawdown` uses all trades within `lookback_period` (in minutes) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` (in minutes) after the last trade - assuming that the bot needs some time to let markets recover. + +The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades within the last 48 candles. + +```json +"protections": [ + { + "method": "MaxDrawdown", + "lookback_period_candles": 48, + "trade_limit": 20, + "stop_duration_candles": 12, + "max_allowed_drawdown": 0.2 + }, +], + +``` + +#### Low Profit Pairs + +`LowProfitPairs` uses all trades for a pair within `lookback_period` (in minutes) to determine the overall profit ratio. +If that ratio is below `required_profit`, that pair will be locked for `stop_duration` (in minutes). + +The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles. + +```json +"protections": [ + { + "method": "LowProfitPairs", + "lookback_period_candles": 6, + "trade_limit": 2, + "stop_duration": 60, + "required_profit": 0.02 + } +], +``` + +#### Cooldown Period + +`CooldownPeriod` locks a pair for `stop_duration` (in minutes) after selling, avoiding a re-entry for this pair for `stop_duration` minutes. + +The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down". + +```json +"protections": [ + { + "method": "CooldownPeriod", + "stop_duration_candles": 2 + } +], +``` + +!!! Note + This Protection applies only at pair-level, and will never lock all pairs globally. + This Protection does not consider `lookback_period` as it only looks at the latest trade. + +### Full example of Protections + +All protections can be combined at will, also with different parameters, creating a increasing wall for under-performing pairs. +All protections are evaluated in the sequence they are defined. + +The below example assumes a timeframe of 1 hour: + +* Locks each pair after selling for an additional 5 candles (`CooldownPeriod`), giving other pairs a chance to get filled. +* Stops trading for 4 hours (`4 * 1h candles`) if the last 2 days (`48 * 1h candles`) had 20 trades, which caused a max-drawdown of more than 20%. (`MaxDrawdown`). +* Stops trading if more than 4 stoploss occur for all pairs within a 1 day (`24 * 1h candles`) limit (`StoplossGuard`). +* Locks all pairs that had 4 Trades within the last 6 hours (`6 * 1h candles`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`). +* Locks all pairs for 2 candles that had a profit of below 0.01 (<1%) within the last 24h (`24 * 1h candles`), a minimum of 4 trades. + +```json +"timeframe": "1h", +"protections": [ + { + "method": "CooldownPeriod", + "stop_duration_candles": 5 + }, + { + "method": "MaxDrawdown", + "lookback_period_candles": 48, + "trade_limit": 20, + "stop_duration_candles": 4, + "max_allowed_drawdown": 0.2 + }, + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 2, + "only_per_pair": false + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 6, + "trade_limit": 2, + "stop_duration_candles": 60, + "required_profit": 0.02 + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 2, + "required_profit": 0.01 + } + ], +``` diff --git a/docs/index.md b/docs/index.md index adc661300..e6882263b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,13 +8,13 @@ Fork -Download +Download Follow @freqtrade ## Introduction -Freqtrade is a crypto-currency algorithmic trading software developed in python (3.6+) and supported on Windows, macOS and Linux. +Freqtrade is a crypto-currency algorithmic trading software developed in python (3.7+) and supported on Windows, macOS and Linux. !!! Danger "DISCLAIMER" This software is for educational purposes only. Do not risk money which you are afraid to lose. USE THE SOFTWARE AT YOUR OWN RISK. THE AUTHORS AND ALL AFFILIATES ASSUME NO RESPONSIBILITY FOR YOUR TRADING RESULTS. @@ -37,13 +37,9 @@ Freqtrade is a crypto-currency algorithmic trading software developed in python ## Requirements -### Up to date clock - -The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges. - ### Hardware requirements -To run this bot we recommend you a cloud instance with a minimum of: +To run this bot we recommend you a linux cloud instance with a minimum of: - 2GB RAM - 1GB disk space @@ -55,7 +51,7 @@ To run this bot we recommend you a cloud instance with a minimum of: Alternatively -- Python 3.6.x +- Python 3.7+ - pip (pip3) - git - TA-Lib @@ -63,11 +59,14 @@ Alternatively ## Support -### Help / Slack -For any questions not covered by the documentation or for further information about the bot, we encourage you to join our passionate Slack community. +### Help / Discord / Slack -Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) to join the Freqtrade Slack channel. +For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. + +Please check out our [discord server](https://discord.gg/MA9v74M). + +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg). ## Ready to try? -Begin by reading our installation guide [for docker](docker.md), or for [installation without docker](installation.md). +Begin by reading our installation guide [for docker](docker_quickstart.md) (recommended), or for [installation without docker](installation.md). diff --git a/docs/installation.md b/docs/installation.md index c03be55d1..be98c45a8 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -10,7 +10,7 @@ Please consider using the prebuilt [docker images](docker.md) to get started qui Click each one for install guide: -* [Python >= 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/) +* [Python >= 3.7.x](http://docs.python-guide.org/en/latest/starting/installation/) * [pip](https://pip.pypa.io/en/stable/installing/) * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * [virtualenv](https://virtualenv.pypa.io/en/stable/installation.html) (Recommended) @@ -18,6 +18,9 @@ Click each one for install guide: We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot), which is optional but recommended. +!!! Warning "Up-to-date clock" + The clock on the system running the bot must be accurate, synchronized to a NTP server frequently enough to avoid problems with communication to the exchanges. + ## Quick start Freqtrade provides the Linux/MacOS Easy Installation script to install all dependencies and help you configure the bot. @@ -28,21 +31,21 @@ Freqtrade provides the Linux/MacOS Easy Installation script to install all depen The easiest way to install and run Freqtrade is to clone the bot Github repository and then run the Easy Installation script, if it's available for your platform. !!! Note "Version considerations" - When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `master` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). + When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `stable` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). !!! Note - Python3.6 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. + Python3.7 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. This can be achieved with the following commands: ```bash git clone https://github.com/freqtrade/freqtrade.git cd freqtrade -git checkout master # Optional, see (1) +# git checkout stable # Optional, see (1) ./setup.sh --install ``` -(1) This command switches the cloned repository to the use of the `master` branch. It's not needed if you wish to stay on the `develop` branch. You may later switch between branches at any time with the `git checkout master`/`git checkout develop` commands. +(1) This command switches the cloned repository to the use of the `stable` branch. It's not needed if you wish to stay on the `develop` branch. You may later switch between branches at any time with the `git checkout stable`/`git checkout develop` commands. ## Easy Installation Script (Linux/MacOS) @@ -53,14 +56,14 @@ $ ./setup.sh usage: -i,--install Install freqtrade from scratch -u,--update Command git pull to update. - -r,--reset Hard reset your develop/master branch. + -r,--reset Hard reset your develop/stable branch. -c,--config Easy config generator (Will override your existing file). ``` ** --install ** With this option, the script will install the bot and most dependencies: -You will need to have git and python3.6+ installed beforehand for this to work. +You will need to have git and python3.7+ installed beforehand for this to work. * Mandatory software as: `ta-lib` * Setup your virtualenv under `.env/` @@ -73,52 +76,59 @@ This option will pull the last version of your current branch and update your vi ** --reset ** -This option will hard reset your branch (only if you are on either `master` or `develop`) and recreate your virtualenv. +This option will hard reset your branch (only if you are on either `stable` or `develop`) and recreate your virtualenv. ** --config ** DEPRECATED - use `freqtrade new-config -c config.json` instead. +### Activate your virtual environment + +Each time you open a new terminal, you must run `source .env/bin/activate`. + ------ ## Custom Installation -We've included/collected install instructions for Ubuntu 16.04, MacOS, and Windows. These are guidelines and your success may vary with other distros. +We've included/collected install instructions for Ubuntu, MacOS, and Windows. These are guidelines and your success may vary with other distros. OS Specific steps are listed first, the [Common](#common) section below is necessary for all systems. !!! Note - Python3.6 or higher and the corresponding pip are assumed to be available. + Python3.7 or higher and the corresponding pip are assumed to be available. -### Linux - Ubuntu 16.04 +=== "Ubuntu/Debian" + #### Install necessary dependencies -#### Install necessary dependencies + ```bash + sudo apt-get update + sudo apt-get install build-essential git + ``` -```bash -sudo apt-get update -sudo apt-get install build-essential git -``` +=== "RaspberryPi/Raspbian" + The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/). + This image comes with python3.7 preinstalled, making it easy to get freqtrade up and running. -### Raspberry Pi / Raspbian + Tested using a Raspberry Pi 3 with the Raspbian Buster lite image, all updates applied. + -The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/) from at least September 2019. -This image comes with python3.7 preinstalled, making it easy to get freqtrade up and running. + ``` bash + sudo apt-get install python3-venv libatlas-base-dev cmake + # Use pywheels.org to speed up installation + sudo echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > tee /etc/pip.conf -Tested using a Raspberry Pi 3 with the Raspbian Buster lite image, all updates applied. + git clone https://github.com/freqtrade/freqtrade.git + cd freqtrade -``` bash -sudo apt-get install python3-venv libatlas-base-dev -git clone https://github.com/freqtrade/freqtrade.git -cd freqtrade + bash setup.sh -i + ``` -bash setup.sh -i -``` + !!! Note "Installation duration" + Depending on your internet speed and the Raspberry Pi version, installation can take multiple hours to complete. + Due to this, we recommend to use the prebuild docker-image for Raspberry, by following the [Docker quickstart documentation](docker_quickstart.md) -!!! Note "Installation duration" - Depending on your internet speed and the Raspberry Pi version, installation can take multiple hours to complete. - -!!! Note - The above does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. - We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerful machine. + !!! Note + The above does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. + We do not advise to run hyperopt on a Raspberry Pi, since this is a very resource-heavy operation, which should be done on powerful machine. ### Common @@ -169,12 +179,7 @@ Clone the git repository: ```bash git clone https://github.com/freqtrade/freqtrade.git cd freqtrade -``` - -Optionally checkout the master branch to get the latest stable release: - -```bash -git checkout master +git checkout stable ``` #### 4. Install python dependencies @@ -212,73 +217,19 @@ On Linux, as an optional post-installation task, you may wish to setup the bot t ------ -## Using Conda +### Anaconda Freqtrade can also be installed using Anaconda (or Miniconda). +!!! Note + This requires the [ta-lib](#1-install-ta-lib) C-library to be installed first. See below. + ``` bash conda env create -f environment.yml ``` -!!! Note - This requires the [ta-lib](#1-install-ta-lib) C-library to be installed first. - -## Windows - -We recommend that Windows users use [Docker](docker.md) as this will work much easier and smoother (also more secure). - -If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work. -If that is not available on your system, feel free to try the instructions below, which led to success for some. - -### Install freqtrade manually - -!!! Note - Make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows. - -!!! Hint - Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Conda section](#using-conda) in this document for more information. - -#### Clone the git repository - -```bash -git clone https://github.com/freqtrade/freqtrade.git -``` - -#### Install ta-lib - -Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). - -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version) - -```cmd ->cd \path\freqtrade-develop ->python -m venv .env ->.env\Scripts\activate.bat -REM optionally install ta-lib from wheel -REM >pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl ->pip install -r requirements.txt ->pip install -e . ->freqtrade -``` - -> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222) - -#### Error during installation under Windows - -``` bash -error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools -``` - -Unfortunately, many packages requiring compilation don't provide a pre-build wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use. - -The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building c code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or [docker](docker.md) first. - ---- - -Now you have an environment ready, the next step is -[Bot Configuration](configuration.md). - -## Troubleshooting +----- +## Troubleshooting ### MacOS installation error @@ -291,4 +242,9 @@ For MacOS 10.14, this can be accomplished with the below command. open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg ``` -If this file is inexistant, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details. +If this file is inexistent, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details. + +----- + +Now you have an environment ready, the next step is +[Bot Configuration](configuration.md). diff --git a/docs/javascripts/config.js b/docs/javascripts/config.js new file mode 100644 index 000000000..95d619efc --- /dev/null +++ b/docs/javascripts/config.js @@ -0,0 +1,12 @@ +window.MathJax = { + tex: { + inlineMath: [["\\(", "\\)"]], + displayMath: [["\\[", "\\]"]], + processEscapes: true, + processEnvironments: true + }, + options: { + ignoreHtmlClass: ".*|", + processHtmlClass: "arithmatex" + } +}; \ No newline at end of file diff --git a/docs/plotting.md b/docs/plotting.md index be83065a6..ed682e44b 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -31,7 +31,7 @@ usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--plot-limit INT] [--db-url PATH] [--trade-source {DB,file}] [--export EXPORT] [--export-filename PATH] - [--timerange TIMERANGE] [-i TICKER_INTERVAL] + [--timerange TIMERANGE] [-i TIMEFRAME] [--no-trades] optional arguments: @@ -65,7 +65,7 @@ optional arguments: _today.json` --timerange TIMERANGE Specify what timerange of data to use. - -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`). --no-trades Skip using trades from backtesting file and DB. @@ -168,6 +168,7 @@ Additional features when using plot_config include: * Specify colors per indicator * Specify additional subplots +* Specify indicator pairs to fill area in between The sample plot configuration below specifies fixed colors for the indicators. Otherwise consecutive plots may produce different colorschemes each time, making comparisons difficult. It also allows multiple subplots to display both MACD and RSI at the same time. @@ -183,23 +184,33 @@ Sample configuration with inline comments explaining the process: 'ema50': {'color': '#CCCCCC'}, # By omitting color, a random color is selected. 'sar': {}, + # fill area between senkou_a and senkou_b + 'senkou_a': { + 'color': 'green', #optional + 'fill_to': 'senkou_b', + 'fill_label': 'Ichimoku Cloud' #optional, + 'fill_color': 'rgba(255,76,46,0.2)', #optional + }, + # plot senkou_b, too. Not only the area to it. + 'senkou_b': {} }, 'subplots': { # Create subplot MACD "MACD": { - 'macd': {'color': 'blue'}, - 'macdsignal': {'color': 'orange'}, + 'macd': {'color': 'blue', 'fill_to': 'macdhist'}, + 'macdsignal': {'color': 'orange'} }, # Additional subplot RSI "RSI": { - 'rsi': {'color': 'red'}, + 'rsi': {'color': 'red'} } } } -``` +``` !!! Note - The above configuration assumes that `ema10`, `ema50`, `macd`, `macdsignal` and `rsi` are columns in the DataFrame created by the strategy. + The above configuration assumes that `ema10`, `ema50`, `senkou_a`, `senkou_b`, + `macd`, `macdsignal`, `macdhist` and `rsi` are columns in the DataFrame created by the strategy. ## Plot profit @@ -224,10 +235,11 @@ Possible options for the `freqtrade plot-profit` subcommand: ``` usage: freqtrade plot-profit [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] + [-d PATH] [--userdir PATH] [-s NAME] + [--strategy-path PATH] [-p PAIRS [PAIRS ...]] [--timerange TIMERANGE] [--export EXPORT] [--export-filename PATH] [--db-url PATH] - [--trade-source {DB,file}] [-i TICKER_INTERVAL] + [--trade-source {DB,file}] [-i TIMEFRAME] optional arguments: -h, --help show this help message and exit @@ -250,7 +262,7 @@ optional arguments: --trade-source {DB,file} Specify the source for trades (Can be DB or file (backtest file)) Default: file - -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL + -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`). @@ -261,14 +273,20 @@ Common arguments: details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH Path to userdata directory. +Strategy arguments: + -s NAME, --strategy NAME + Specify strategy class name which will be used by the + bot. + --strategy-path PATH Specify additional strategy lookup path. ``` The `-p/--pairs` argument, can be used to limit the pairs that are considered for this calculation. @@ -278,7 +296,7 @@ Examples: Use custom backtest-export file ``` bash -freqtrade plot-profit -p LTC/BTC --export-filename user_data/backtest_results/backtest-result-Strategy005.json +freqtrade plot-profit -p LTC/BTC --export-filename user_data/backtest_results/backtest-result.json ``` Use custom database diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 000000000..1f785bbaa --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,3 @@ +# Plugins +--8<-- "includes/pairlists.md" +--8<-- "includes/protections.md" diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index b8ea338de..2b133cb07 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,2 +1,3 @@ -mkdocs-material==5.2.3 +mkdocs-material==6.1.7 mdx_truly_sane_lists==1.2 +pymdown-extensions==8.0.1 diff --git a/docs/rest-api.md b/docs/rest-api.md index ed5f355b4..9bb35ce91 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -13,6 +13,7 @@ Sample configuration: "listen_port": 8080, "verbosity": "info", "jwt_secret_key": "somethingrandom", + "CORS_origins": [], "username": "Freqtrader", "password": "SuperSecret1!" }, @@ -45,7 +46,7 @@ secrets.token_hex() ### Configuration with docker -If you run your bot using docker, you'll need to have the bot listen to incomming connections. The security is then handled by docker. +If you run your bot using docker, you'll need to have the bot listen to incoming connections. The security is then handled by docker. ``` json "api_server": { @@ -103,28 +104,43 @@ By default, the script assumes `127.0.0.1` (localhost) and port `8080` to be use python3 scripts/rest_client.py --config rest_config.json [optional parameters] ``` -## Available commands +## Available endpoints -| Command | Default | Description | -|----------|---------|-------------| -| `start` | | Starts the trader -| `stop` | | Stops the trader -| `stopbuy` | | Stops the trader from opening new trades. Gracefully closes open trades according to their rules. -| `reload_conf` | | Reloads the configuration file -| `show_config` | | Shows part of the current configuration with relevant settings to operation -| `status` | | Lists all open trades -| `count` | | Displays number of trades used and available -| `profit` | | Display a summary of your profit/loss from close trades and some stats about your performance -| `forcesell ` | | Instantly sells the given trade (Ignoring `minimum_roi`). -| `forcesell all` | | Instantly sells all open trades (Ignoring `minimum_roi`). -| `forcebuy [rate]` | | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) -| `performance` | | Show performance of each finished trade grouped by pair -| `balance` | | Show account balance per currency -| `daily ` | 7 | Shows profit or loss per day, over the last n days -| `whitelist` | | Show the current whitelist -| `blacklist [pair]` | | Show the current blacklist, or adds a pair to the blacklist. -| `edge` | | Show validated pairs by Edge if it is enabled. -| `version` | | Show version +| Command | Description | +|----------|-------------| +| `ping` | Simple command testing the API Readiness - requires no authentication. +| `start` | Starts the trader. +| `stop` | Stops the trader. +| `stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules. +| `reload_config` | Reloads the configuration file. +| `trades` | List last trades. +| `delete_trade ` | Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange. +| `show_config` | Shows part of the current configuration with relevant settings to operation. +| `logs` | Shows last log messages. +| `status` | Lists all open trades. +| `count` | Displays number of trades used and available. +| `locks` | Displays currently locked pairs. +| `profit` | Display a summary of your profit/loss from close trades and some stats about your performance. +| `forcesell ` | Instantly sells the given trade (Ignoring `minimum_roi`). +| `forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`). +| `forcebuy [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) +| `performance` | Show performance of each finished trade grouped by pair. +| `balance` | Show account balance per currency. +| `daily ` | Shows profit or loss per day, over the last n days (n defaults to 7). +| `stats` | Display a summary of profit / loss reasons as well as average holding times. +| `whitelist` | Show the current whitelist. +| `blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist. +| `edge` | Show validated pairs by Edge if it is enabled. +| `pair_candles` | Returns dataframe for a pair / timeframe combination while the bot is running. **Alpha** +| `pair_history` | Returns an analyzed dataframe for a given timerange, analyzed by a given strategy. **Alpha** +| `plot_config` | Get plot config from the strategy (or nothing if not configured). **Alpha** +| `strategies` | List strategies in strategy directory. **Alpha** +| `strategy ` | Get specific Strategy content. **Alpha** +| `available_pairs` | List available backtest data. **Alpha** +| `version` | Show version. + +!!! Warning "Alpha status" + Endpoints labeled with *Alpha status* above may change at any time without notice. Possible commands can be listed from the rest-client script using the `help` command. @@ -134,78 +150,117 @@ python3 scripts/rest_client.py help ``` output Possible commands: + +available_pairs + Return available pair (backtest data) based on timeframe / stake_currency selection + + :param timeframe: Only pairs with this timeframe available. + :param stake_currency: Only pairs that include this timeframe + balance - Get the account balance - :returns: json object + Get the account balance. blacklist - Show the current blacklist + Show the current blacklist. + :param add: List of coins to add (example: "BNB/BTC") - :returns: json object count - Returns the amount of open trades - :returns: json object + Return the amount of open trades. daily - Returns the amount of open trades - :returns: json object + Return the amount of open trades. + +delete_trade + Delete trade from the database. + Tries to close open orders. Requires manual handling of this asset on the exchange. + + :param trade_id: Deletes the trade with this ID from the database. edge - Returns information about edge - :returns: json object + Return information about edge. forcebuy - Buy an asset + Buy an asset. + :param pair: Pair to buy (ETH/BTC) :param price: Optional - price to buy - :returns: json object of the trade forcesell - Force-sell a trade + Force-sell a trade. + :param tradeid: Id of the trade (can be received via status command) - :returns: json object + +logs + Show latest logs. + + :param limit: Limits log messages to the last logs. No limit to get all the trades. + +pair_candles + Return live dataframe for . + + :param pair: Pair to get data for + :param timeframe: Only pairs with this timeframe available. + :param limit: Limit result to the last n candles. + +pair_history + Return historic, analyzed dataframe + + :param pair: Pair to get data for + :param timeframe: Only pairs with this timeframe available. + :param strategy: Strategy to analyze and get values for + :param timerange: Timerange to get data for (same format than --timerange endpoints) performance - Returns the performance of the different coins - :returns: json object + Return the performance of the different coins. + +plot_config + Return plot configuration if the strategy defines one. profit - Returns the profit summary - :returns: json object + Return the profit summary. -reload_conf - Reload configuration - :returns: json object +reload_config + Reload configuration. show_config + Returns part of the configuration, relevant for trading operations. - :return: json object containing the version start - Start the bot if it's in stopped state. - :returns: json object + Start the bot if it's in the stopped state. + +stats + Return the stats report (durations, sell-reasons). status - Get the status of open trades - :returns: json object + Get the status of open trades. stop - Stop the bot. Use start to restart - :returns: json object + Stop the bot. Use `start` to restart. stopbuy - Stop buying (but handle sells gracefully). - use reload_conf to reset - :returns: json object + Stop buying (but handle sells gracefully). Use `reload_config` to reset. + +strategies + Lists available strategies + +strategy + Get strategy details + + :param strategy: Strategy class name + +trades + Return trades history. + + :param limit: Limits trades to the X last trades. No limit to get all the trades. version - Returns the version of the bot - :returns: json object containing the version + Return the version of the bot. whitelist - Show the current whitelist - :returns: json object + Show the current whitelist. + ``` ## Advanced API usage using JWT tokens @@ -232,3 +287,26 @@ Since the access token has a short timeout (15 min) - the `token/refresh` reques > curl -X POST --header "Authorization: Bearer ${refresh_token}"http://localhost:8080/api/v1/token/refresh {"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk5NzQsIm5iZiI6MTU4OTExOTk3NCwianRpIjoiMDBjNTlhMWUtMjBmYS00ZTk0LTliZjAtNWQwNTg2MTdiZDIyIiwiZXhwIjoxNTg5MTIwODc0LCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.1seHlII3WprjjclY6DpRhen0rqdF4j6jbvxIhUFaSbs"} ``` + +## CORS + +All web-based frontends are subject to [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) - Cross-Origin Resource Sharing. +Since most of the requests to the Freqtrade API must be authenticated, a proper CORS policy is key to avoid security problems. +Also, the standard disallows `*` CORS policies for requests with credentials, so this setting must be set appropriately. + +Users can configure this themselves via the `CORS_origins` configuration setting. +It consists of a list of allowed sites that are allowed to consume resources from the bot's API. + +Assuming your application is deployed as `https://frequi.freqtrade.io/home/` - this would mean that the following configuration becomes necessary: + +```jsonc +{ + //... + "jwt_secret_key": "somethingrandom", + "CORS_origins": ["https://frequi.freqtrade.io"], + //... +} +``` + +!!! Note + We strongly recommend to also set `jwt_secret_key` to something random and known only to yourself to avoid unauthorized access to your bot. diff --git a/docs/sandbox-testing.md b/docs/sandbox-testing.md index 7f3457d15..9c14412de 100644 --- a/docs/sandbox-testing.md +++ b/docs/sandbox-testing.md @@ -1,104 +1,59 @@ # Sandbox API testing -Where an exchange provides a sandbox for risk-free integration, or end-to-end, testing CCXT provides access to these. +Some exchanges provide sandboxes or testbeds for risk-free testing, while running the bot against a real exchange. +With some configuration, freqtrade (in combination with ccxt) provides access to these. -This document is a *light overview of configuring Freqtrade and GDAX sandbox. -This can be useful to developers and trader alike as Freqtrade is quite customisable. +This document is an overview to configure Freqtrade to be used with sandboxes. +This can be useful to developers and trader alike. -When testing your API connectivity, make sure to use the following URLs. -***Website** -https://public.sandbox.gdax.com -***REST API** -https://api-public.sandbox.gdax.com +## Exchanges known to have a sandbox / testnet + +* [binance](https://testnet.binance.vision/) +* [coinbasepro](https://public.sandbox.pro.coinbase.com) +* [gemini](https://exchange.sandbox.gemini.com/) +* [huobipro](https://www.testnet.huobi.pro/) +* [kucoin](https://sandbox.kucoin.com/) +* [phemex](https://testnet.phemex.com/) + +!!! Note + We did not test correct functioning of all of the above testnets. Please report your experiences with each sandbox. --- -# Configure a Sandbox account on Gdax +## Configure a Sandbox account -Aim of this document section +When testing your API connectivity, make sure to use the appropriate sandbox / testnet URL. -- An sanbox account -- create 2FA (needed to create an API) -- Add test 50BTC to account -- Create : -- - API-KEY -- - API-Secret -- - API Password +In general, you should follow these steps to enable an exchange's sandbox: -## Acccount +* Figure out if an exchange has a sandbox (most likely by using google or the exchange's support documents) +* Create a sandbox account (often the sandbox-account requires separate registration) +* [Add some test assets to account](#add-test-funds) +* Create API keys -This link will redirect to the sandbox main page to login / create account dialogues: -https://public.sandbox.pro.coinbase.com/orders/ +### Add test funds -After registration and Email confimation you wil be redirected into your sanbox account. It is easy to verify you're in sandbox by checking the URL bar. -> https://public.sandbox.pro.coinbase.com/ +Usually, sandbox exchanges allow depositing funds directly via web-interface. +You should make sure to have a realistic amount of funds available to your test-account, so results are representable of your real account funds. -## Enable 2Fa (a prerequisite to creating sandbox API Keys) +!!! Warning + Test exchanges will **NEVER** require your real credit card or banking details! -From within sand box site select your profile, top right. ->Or as a direct link: https://public.sandbox.pro.coinbase.com/profile +## Configure freqtrade to use a exchange's sandbox -From the menu panel to the left of the screen select - -> Security: "*View or Update*" - -In the new site select "enable authenticator" as typical google Authenticator. - -- open Google Authenticator on your phone -- scan barcode -- enter your generated 2fa - -## Enable API Access - -From within sandbox select profile>api>create api-keys ->or as a direct link: https://public.sandbox.pro.coinbase.com/profile/api - -Click on "create one" and ensure **view** and **trade** are "checked" and sumbit your 2FA - -- **Copy and paste the Passphase** into a notepade this will be needed later -- **Copy and paste the API Secret** popup into a notepad this will needed later -- **Copy and paste the API Key** into a notepad this will needed later - -## Add 50 BTC test funds - -To add funds, use the web interface deposit and withdraw buttons. - -To begin select 'Wallets' from the top menu. -> Or as a direct link: https://public.sandbox.pro.coinbase.com/wallets - -- Deposits (bottom left of screen) -- - Deposit Funds Bitcoin -- - - Coinbase BTC Wallet -- - - - Max (50 BTC) -- - - - - Deposit - -*This process may be repeated for other currencies, ETH as example* - ---- - -# Configure Freqtrade to use Gax Sandbox - -The aim of this document section - -- Enable sandbox URLs in Freqtrade -- Configure API -- - secret -- - key -- - passphrase - -## Sandbox URLs +### Sandbox URLs Freqtrade makes use of CCXT which in turn provides a list of URLs to Freqtrade. These include `['test']` and `['api']`. -- `[Test]` if available will point to an Exchanges sandbox. -- `[Api]` normally used, and resolves to live API target on the exchange +* `[Test]` if available will point to an Exchanges sandbox. +* `[Api]` normally used, and resolves to live API target on the exchange. To make use of sandbox / test add "sandbox": true, to your config.json ```json "exchange": { - "name": "gdax", + "name": "coinbasepro", "sandbox": true, "key": "5wowfxemogxeowo;heiohgmd", "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==", @@ -106,36 +61,57 @@ To make use of sandbox / test add "sandbox": true, to your config.json "outdated_offset": 5 "pair_whitelist": [ "BTC/USD" + ] + }, + "datadir": "user_data/data/coinbasepro_sandbox" ``` -Also insert your +Also the following information: -- api-key (noted earlier) -- api-secret (noted earlier) -- password (the passphrase - noted earlier) +* api-key (created for the sandbox webpage) +* api-secret (noted earlier) +* password (the passphrase - noted earlier) + +!!! Tip "Different data directory" + We also recommend to set `datadir` to something identifying downloaded data as sandbox data, to avoid having sandbox data mixed with data from the real exchange. + This can be done by adding the `"datadir"` key to the configuration. + Now, whenever you use this configuration, your data directory will be set to this directory. --- ## You should now be ready to test your sandbox -Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox. -** Typically the BTC/USD has the most activity in sandbox to test against. +Ensure Freqtrade logs show the sandbox URL, and trades made are shown in sandbox. Also make sure to select a pair which shows at least some decent value (which very often is BTC/). -## GDAX - Old Candles problem +## Common problems with sandbox exchanges -It is my experience that GDAX sandbox candles may be 20+- minutes out of date. This can cause trades to fail as one of Freqtrades safety checks. +Sandbox exchange instances often have very low volume, which can cause some problems which usually are not seen on a real exchange instance. -To disable this check, add / change the `"outdated_offset"` parameter in the exchange section of your configuration to adjust for this delay. -Example based on the above configuration: +### Old Candles problem -```json - "exchange": { - "name": "gdax", - "sandbox": true, - "key": "5wowfxemogxeowo;heiohgmd", - "secret": "/ZMH1P62rCVmwefewrgcewX8nh4gob+lywxfwfxwwfxwfNsH1ySgvWCUR/w==", - "password": "1bkjfkhfhfu6sr", - "outdated_offset": 30 - "pair_whitelist": [ - "BTC/USD" -``` +Since Sandboxes often have low volume, candles can be quite old and show no volume. +To disable the error "Outdated history for pair ...", best increase the parameter `"outdated_offset"` to a number that seems realistic for the sandbox you're using. + +### Unfilled orders + +Sandboxes often have very low volumes - which means that many trades can go unfilled, or can go unfilled for a very long time. + +To mitigate this, you can try to match the first order on the opposite orderbook side using the following configuration: + +``` jsonc + "order_types": { + "buy": "limit", + "sell": "limit" + // ... + }, + "bid_strategy": { + "price_side": "ask", + // ... + }, + "ask_strategy":{ + "price_side": "bid", + // ... + }, + ``` + + The configuration is similar to the suggested configuration for market orders - however by using limit-orders you can avoid moving the price too much, and you can set the worst price you might get. diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index 895a0536a..569af33ff 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -13,6 +13,15 @@ Feel free to use a visual Database editor like SqliteBrowser if you feel more co sudo apt-get install sqlite3 ``` +### Using sqlite3 via docker-compose + +The freqtrade docker image does contain sqlite3, so you can edit the database without having to install anything on the host system. + +``` bash +docker-compose exec freqtrade /bin/bash +sqlite3 .sqlite +``` + ## Open the DB ```bash @@ -34,52 +43,6 @@ sqlite3 .schema ``` -### Trade table structure - -```sql -CREATE TABLE trades - id INTEGER NOT NULL, - exchange VARCHAR NOT NULL, - pair VARCHAR NOT NULL, - is_open BOOLEAN NOT NULL, - fee_open FLOAT NOT NULL, - fee_open_cost FLOAT, - fee_open_currency VARCHAR, - fee_close FLOAT NOT NULL, - fee_close_cost FLOAT, - fee_close_currency VARCHAR, - open_rate FLOAT, - open_rate_requested FLOAT, - open_trade_price FLOAT, - close_rate FLOAT, - close_rate_requested FLOAT, - close_profit FLOAT, - close_profit_abs FLOAT, - stake_amount FLOAT NOT NULL, - amount FLOAT, - open_date DATETIME NOT NULL, - close_date DATETIME, - open_order_id VARCHAR, - stop_loss FLOAT, - stop_loss_pct FLOAT, - initial_stop_loss FLOAT, - initial_stop_loss_pct FLOAT, - stoploss_order_id VARCHAR, - stoploss_last_update DATETIME, - max_rate FLOAT, - min_rate FLOAT, - sell_reason VARCHAR, - strategy VARCHAR, - ticker_interval INTEGER, - PRIMARY KEY (id), - CHECK (is_open IN (0, 1)) -); -CREATE INDEX ix_trades_stoploss_order_id ON trades (stoploss_order_id); -CREATE INDEX ix_trades_pair ON trades (pair); -CREATE INDEX ix_trades_is_open ON trades (is_open); - -``` - ## Get all trades in the table ```sql @@ -89,19 +52,19 @@ SELECT * FROM trades; ## Fix trade still open after a manual sell on the exchange !!! Warning - Manually selling a pair on the exchange will not be detected by the bot and it will try to sell anyway. Whenever possible, forcesell should be used to accomplish the same thing. - It is strongly advised to backup your database file before making any manual changes. + Manually selling a pair on the exchange will not be detected by the bot and it will try to sell anyway. Whenever possible, forcesell should be used to accomplish the same thing. + It is strongly advised to backup your database file before making any manual changes. !!! Note - This should not be necessary after /forcesell, as forcesell orders are closed automatically by the bot on the next iteration. + This should not be necessary after /forcesell, as forcesell orders are closed automatically by the bot on the next iteration. ```sql UPDATE trades SET is_open=0, close_date=, close_rate=, - close_profit=close_rate/open_rate-1, - close_profit_abs = (amount * * (1 - fee_close) - (amount * open_rate * 1 - fee_open), + close_profit = close_rate / open_rate - 1, + close_profit_abs = (amount * * (1 - fee_close) - (amount * (open_rate * (1 - fee_open)))), sell_reason= WHERE id=; ``` @@ -111,24 +74,28 @@ WHERE id=; ```sql UPDATE trades SET is_open=0, - close_date='2017-12-20 03:08:45.103418', + close_date='2020-06-20 03:08:45.103418', close_rate=0.19638016, close_profit=0.0496, - close_profit_abs = (amount * 0.19638016 * (1 - fee_close) - (amount * open_rate * 1 - fee_open) + close_profit_abs = (amount * 0.19638016 * (1 - fee_close) - (amount * (open_rate * (1 - fee_open)))), sell_reason='force_sell' WHERE id=31; ``` -## Insert manually a new trade +## Remove trade from the database + +!!! Tip "Use RPC Methods to delete trades" + Consider using `/delete ` via telegram or rest API. That's the recommended way to deleting trades. + +If you'd still like to remove a trade from the database directly, you can use the below query. ```sql -INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) -VALUES ('bittrex', 'ETH/BTC', 1, 0.0025, 0.0025, , , , '') +DELETE FROM trades WHERE id = ; ``` -##### Example: - ```sql -INSERT INTO trades (exchange, pair, is_open, fee_open, fee_close, open_rate, stake_amount, amount, open_date) -VALUES ('bittrex', 'ETH/BTC', 1, 0.0025, 0.0025, 0.00258580, 0.002, 0.7715262081, '2017-11-28 12:44:24.000000') +DELETE FROM trades WHERE id = 31; ``` + +!!! Warning + This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause. diff --git a/docs/stoploss.md b/docs/stoploss.md index 0e43817ec..1e21fc50d 100644 --- a/docs/stoploss.md +++ b/docs/stoploss.md @@ -6,7 +6,69 @@ For example, value `-0.10` will cause immediate sell if the profit dips below -1 Most of the strategy files already include the optimal `stoploss` value. !!! Info - All stoploss properties mentioned in this file can be set in the Strategy, or in the configuration. Configuration values will override the strategy values. + All stoploss properties mentioned in this file can be set in the Strategy, or in the configuration. + Configuration values will override the strategy values. + +## Stop Loss On-Exchange/Freqtrade + +Those stoploss modes can be *on exchange* or *off exchange*. + +These modes can be configured with these values: + +``` python + 'emergencysell': 'market', + 'stoploss_on_exchange': False + 'stoploss_on_exchange_interval': 60, + 'stoploss_on_exchange_limit_ratio': 0.99 +``` + +!!! Note + Stoploss on exchange is only supported for Binance (stop-loss-limit), Kraken (stop-loss-market, stop-loss-limit) and FTX (stop limit and stop-market) as of now. + Do not set too low/tight stoploss value if using stop loss on exchange! + If set to low/tight then you have greater risk of missing fill on the order and stoploss will not work. + +### stoploss_on_exchange and stoploss_on_exchange_limit_ratio + +Enable or Disable stop loss on exchange. +If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled. + +If `stoploss_on_exchange` uses limit orders, the exchange needs 2 prices, the stoploss_price and the Limit price. +`stoploss` defines the stop-price where the limit order is placed - and limit should be slightly below this. +If an exchange supports both limit and market stoploss orders, then the value of `stoploss` will be used to determine the stoploss type. + +Calculation example: we bought the asset at 100\$. +Stop-price is 95\$, then limit would be `95 * 0.99 = 94.05$` - so the limit order fill can happen between 95$ and 94.05$. + +For example, assuming the stoploss is on exchange, and trailing stoploss is enabled, and the market is going up, then the bot automatically cancels the previous stoploss order and puts a new one with a stop value higher than the previous stoploss order. + +!!! Note + If `stoploss_on_exchange` is enabled and the stoploss is cancelled manually on the exchange, then the bot will create a new stoploss order. + +### stoploss_on_exchange_interval + +In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary. +The bot cannot do these every 5 seconds (at each iteration), otherwise it would get banned by the exchange. +So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute). +This same logic will reapply a stoploss order on the exchange should you cancel it accidentally. + +### emergencysell + +`emergencysell` is an optional value, which defaults to `market` and is used when creating stop loss on exchange orders fails. +The below is the default which is used if not changed in strategy or configuration file. + +Example from strategy file: + +``` python +order_types = { + 'buy': 'limit', + 'sell': 'limit', + 'emergencysell': 'market', + 'stoploss': 'market', + 'stoploss_on_exchange': True, + 'stoploss_on_exchange_interval': 60, + 'stoploss_on_exchange_limit_ratio': 0.99 +} +``` ## Stop Loss Types @@ -17,29 +79,30 @@ At this stage the bot contains the following stoploss support modes: 3. Trailing stop loss, custom positive loss. 4. Trailing stop loss only once the trade has reached a certain offset. -Those stoploss modes can be *on exchange* or *off exchange*. If the stoploss is *on exchange* it means a stoploss limit order is placed on the exchange immediately after buy order happens successfully. This will protect you against sudden crashes in market as the order will be in the queue immediately and if market goes down then the order has more chance of being fulfilled. - -In case of stoploss on exchange there is another parameter called `stoploss_on_exchange_interval`. This configures the interval in seconds at which the bot will check the stoploss and update it if necessary. - -For example, assuming the stoploss is on exchange, and trailing stoploss is enabled, and the market is going up, then the bot automatically cancels the previous stoploss order and puts a new one with a stop value higher than the previous stoploss order. -The bot cannot do this every 5 seconds (at each iteration), otherwise it would get banned by the exchange. -So this parameter will tell the bot how often it should update the stoploss order. The default value is 60 (1 minute). -This same logic will reapply a stoploss order on the exchange should you cancel it accidentally. - -!!! Note - Stoploss on exchange is only supported for Binance (stop-loss-limit) and Kraken (stop-loss-market) as of now. - -## Static Stop Loss +### Static Stop Loss This is very simple, you define a stop loss of x (as a ratio of price, i.e. x * 100% of price). This will try to sell the asset once the loss exceeds the defined loss. -## Trailing Stop Loss +Example of stop loss: + +``` python + stoploss = -0.10 +``` + +For example, simplified math: + +* the bot buys an asset at a price of 100$ +* the stop loss is defined at -10% +* the stop loss would get triggered once the asset drops below 90$ + +### Trailing Stop Loss The initial value for this is `stoploss`, just as you would define your static Stop loss. To enable trailing stoploss: ``` python -trailing_stop = True + stoploss = -0.10 + trailing_stop = True ``` This will now activate an algorithm, which automatically moves the stop loss up every time the price of your asset increases. @@ -47,35 +110,43 @@ This will now activate an algorithm, which automatically moves the stop loss up For example, simplified math: * the bot buys an asset at a price of 100$ -* the stop loss is defined at 2% -* the stop loss would get triggered once the asset dropps below 98$ +* the stop loss is defined at -10% +* the stop loss would get triggered once the asset drops below 90$ * assuming the asset now increases to 102$ -* the stop loss will now be 2% of 102$ or 99.96$ -* now the asset drops in value to 101$, the stop loss will still be 99.96$ and would trigger at 99.96$. +* the stop loss will now be -10% of 102$ = 91.8$ +* now the asset drops in value to 101\$, the stop loss will still be 91.8$ and would trigger at 91.8$. -In summary: The stoploss will be adjusted to be always be 2% of the highest observed price. +In summary: The stoploss will be adjusted to be always be -10% of the highest observed price. -### Custom positive stoploss +### Trailing stop loss, custom positive loss -It is also possible to have a default stop loss, when you are in the red with your buy, but once your profit surpasses a certain percentage, the system will utilize a new stop loss, which can have a different value. -For example your default stop loss is 5%, but once you have 1.1% profit, it will be changed to be only a 1% stop loss, which trails the green candles until it goes below them. +It is also possible to have a default stop loss, when you are in the red with your buy (buy - fee), but once you hit positive result the system will utilize a new stop loss, which can have a different value. +For example, your default stop loss is -10%, but once you have more than 0% profit (example 0.1%) a different trailing stoploss will be used. -Both values require `trailing_stop` to be set to true. +!!! Note + If you want the stoploss to only be changed when you break even of making a profit (what most users want) please refer to next section with [offset enabled](#Trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). + +Both values require `trailing_stop` to be set to true and `trailing_stop_positive` with a value. ``` python - trailing_stop_positive = 0.01 - trailing_stop_positive_offset = 0.011 + stoploss = -0.10 + trailing_stop = True + trailing_stop_positive = 0.02 ``` -The 0.01 would translate to a 1% stop loss, once you hit 1.1% profit. +For example, simplified math: + +* the bot buys an asset at a price of 100$ +* the stop loss is defined at -10% +* the stop loss would get triggered once the asset drops below 90$ +* assuming the asset now increases to 102$ +* the stop loss will now be -2% of 102$ = 99.96$ (99.96$ stop loss will be locked in and will follow asset price increments with -2%) +* now the asset drops in value to 101\$, the stop loss will still be 99.96$ and would trigger at 99.96$ + +The 0.02 would translate to a -2% stop loss. Before this, `stoploss` is used for the trailing stoploss. -Read the [next section](#trailing-only-once-offset-is-reached) to keep stoploss at 5% of the entry point. - -!!! Tip - Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade. - -### Trailing only once offset is reached +### Trailing stop loss only once the trade has reached a certain offset It is also possible to use a static stoploss until the offset is reached, and then trail the trade to take profits once the market turns. @@ -84,24 +155,35 @@ This option can be used with or without `trailing_stop_positive`, but uses `trai ``` python trailing_stop_positive_offset = 0.011 - trailing_only_offset_is_reached = true + trailing_only_offset_is_reached = True ``` -Simplified example: +Configuration (offset is buy-price + 3%): ``` python - stoploss = 0.05 + stoploss = -0.10 + trailing_stop = True + trailing_stop_positive = 0.02 trailing_stop_positive_offset = 0.03 trailing_only_offset_is_reached = True ``` +For example, simplified math: + * the bot buys an asset at a price of 100$ -* the stop loss is defined at 5% -* the stop loss will remain at 95% until profit reaches +3% +* the stop loss is defined at -10% +* the stop loss would get triggered once the asset drops below 90$ +* stoploss will remain at 90$ unless asset increases to or above our configured offset +* assuming the asset now increases to 103$ (where we have the offset configured) +* the stop loss will now be -2% of 103$ = 100.94$ +* now the asset drops in value to 101\$, the stop loss will still be 100.94$ and would trigger at 100.94$ + +!!! Tip + Make sure to have this value (`trailing_stop_positive_offset`) lower than minimal ROI, otherwise minimal ROI will apply first and sell the trade. ## Changing stoploss on open trades -A stoploss on an open trade can be changed by changing the value in the configuration or strategy and use the `/reload_conf` command (alternatively, completely stopping and restarting the bot also works). +A stoploss on an open trade can be changed by changing the value in the configuration or strategy and use the `/reload_config` command (alternatively, completely stopping and restarting the bot also works). The new stoploss value will be applied to open trades (and corresponding log-messages will be generated). diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index 69e2256a1..359280694 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -1,7 +1,12 @@ # Advanced Strategies This page explains some advanced concepts available for strategies. -If you're just getting started, please be familiar with the methods described in the [Strategy Customization](strategy-customization.md) documentation first. +If you're just getting started, please be familiar with the methods described in the [Strategy Customization](strategy-customization.md) documentation and with the [Freqtrade basics](bot-basics.md) first. + +[Freqtrade basics](bot-basics.md) describes in which sequence each method described below is called, which can be helpful to understand which method to use for your custom needs. + +!!! Note + All callback methods described below should only be implemented in a strategy if they are actually used. ## Custom order timeout rules @@ -89,3 +94,129 @@ class Awesomestrategy(IStrategy): return True return False ``` + +## Bot loop start callback + +A simple callback which is called once at the start of every bot throttling iteration. +This can be used to perform calculations which are pair independent (apply to all pairs), loading of external data, etc. + +``` python +import requests + +class Awesomestrategy(IStrategy): + + # ... populate_* methods + + def bot_loop_start(self, **kwargs) -> None: + """ + Called at the start of the bot iteration (one loop). + Might be used to perform pair-independent tasks + (e.g. gather some remote resource for comparison) + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + """ + if self.config['runmode'].value in ('live', 'dry_run'): + # Assign this to the class by using self.* + # can then be used by populate_* methods + self.remote_data = requests.get('https://some_remote_source.example.com') + +``` + +## Bot order confirmation + +### Trade entry (buy order) confirmation + +`confirm_trade_entry()` can be used to abort a trade entry at the latest second (maybe because the price is not what we expect). + +``` python +class Awesomestrategy(IStrategy): + + # ... populate_* methods + + def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, + time_in_force: str, **kwargs) -> bool: + """ + Called right before placing a buy order. + Timing for this function is critical, so avoid doing heavy computations or + network requests in this method. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns True (always confirming). + + :param pair: Pair that's about to be bought. + :param order_type: Order type (as configured in order_types). usually limit or market. + :param amount: Amount in target (quote) currency that's going to be traded. + :param rate: Rate that's going to be used when using limit orders + :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the buy-order is placed on the exchange. + False aborts the process + """ + return True + +``` + +### Trade exit (sell order) confirmation + +`confirm_trade_exit()` can be used to abort a trade exit (sell) at the latest second (maybe because the price is not what we expect). + +``` python +from freqtrade.persistence import Trade + + +class Awesomestrategy(IStrategy): + + # ... populate_* methods + + def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float, + rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool: + """ + Called right before placing a regular sell order. + Timing for this function is critical, so avoid doing heavy computations or + network requests in this method. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns True (always confirming). + + :param pair: Pair that's about to be sold. + :param order_type: Order type (as configured in order_types). usually limit or market. + :param amount: Amount in quote currency. + :param rate: Rate that's going to be used when using limit orders + :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param sell_reason: Sell reason. + Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', + 'sell_signal', 'force_sell', 'emergency_sell'] + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the sell-order is placed on the exchange. + False aborts the process + """ + if sell_reason == 'force_sell' and trade.calc_profit_ratio(rate) < 0: + # Reject force-sells with negative profit + # This is just a sample, please adjust to your needs + # (this does not necessarily make sense, assuming you know when you're force-selling) + return False + return True + +``` + +## Derived strategies + +The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched: + +``` python +class MyAwesomeStrategy(IStrategy): + ... + stoploss = 0.13 + trailing_stop = False + # All other attributes and methods are here as they + # should be in any custom strategy... + ... + +class MyAwesomeStrategy2(MyAwesomeStrategy): + # Override something + stoploss = 0.08 + trailing_stop = True +``` + +Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 7197b0fba..ab64d3a67 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -1,6 +1,8 @@ # Strategy Customization -This page explains where to customize your strategies, and add new indicators. +This page explains how to customize your strategies, add new indicators and set up trading rules. + +Please familiarize yourself with [Freqtrade basics](bot-basics.md) first, which provides overall info on how the bot operates. ## Install a custom strategy file @@ -56,12 +58,12 @@ file as reference.** !!! Note "Strategies and Backtesting" To avoid problems and unexpected differences between Backtesting and dry/live modes, please be aware - that during backtesting the full time-interval is passed to the `populate_*()` methods at once. + that during backtesting the full time range is passed to the `populate_*()` methods at once. It is therefore best to use vectorized operations (across the whole dataframe, not loops) and avoid index referencing (`df.iloc[-1]`), but instead use `df.shift()` to get to the previous candle. !!! Warning "Warning: Using future data" - Since backtesting passes the full time interval to the `populate_*()` methods, the strategy author + Since backtesting passes the full time range to the `populate_*()` methods, the strategy author needs to take care to avoid having the strategy utilize data from the future. Some common patterns for this are listed in the [Common Mistakes](#common-mistakes-when-developing-strategies) section of this document. @@ -139,13 +141,13 @@ By letting the bot know how much history is needed, backtest trades can start at #### Example -Let's try to backtest 1 month (January 2019) of 5m candles using the an example strategy with EMA100, as above. +Let's try to backtest 1 month (January 2019) of 5m candles using an example strategy with EMA100, as above. ``` bash -freqtrade backtesting --timerange 20190101-20190201 --ticker-interval 5m +freqtrade backtesting --timerange 20190101-20190201 --timeframe 5m ``` -Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2019-12-31 15:30:00. +Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2018-12-31 15:30:00. If this data is available, indicators will be calculated with this extended timerange. The instable startup period (up to 2019-01-01 00:00:00) will then be removed before starting backtesting. !!! Note @@ -248,20 +250,20 @@ minimal_roi = { While technically not completely disabled, this would sell once the trade reaches 10000% Profit. -To use times based on candle duration (ticker_interval or timeframe), the following snippet can be handy. -This will allow you to change the ticket_interval for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...) +To use times based on candle duration (timeframe), the following snippet can be handy. +This will allow you to change the timeframe for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...) ``` python from freqtrade.exchange import timeframe_to_minutes class AwesomeStrategy(IStrategy): - ticker_interval = "1d" - ticker_interval_mins = timeframe_to_minutes(ticker_interval) + timeframe = "1d" + timeframe_mins = timeframe_to_minutes(timeframe) minimal_roi = { "0": 0.05, # 5% for the first 3 candles - str(ticker_interval_mins * 3)): 0.02, # 2% after 3 candles - str(ticker_interval_mins * 6)): 0.01, # 1% After 6 candles + str(timeframe_mins * 3)): 0.02, # 2% after 3 candles + str(timeframe_mins * 6)): 0.01, # 1% After 6 candles } ``` @@ -283,14 +285,14 @@ If your exchange supports it, it's recommended to also set `"stoploss_on_exchang For more information on order_types please look [here](configuration.md#understand-order_types). -### Timeframe (ticker interval) +### Timeframe (formerly ticker interval) This is the set of candles the bot should download and use for the analysis. Common values are `"1m"`, `"5m"`, `"15m"`, `"1h"`, however all values supported by your exchange should work. Please note that the same buy/sell signals may work well with one timeframe, but not with the others. -This setting is accessible within the strategy methods as the `self.ticker_interval` attribute. +This setting is accessible within the strategy methods as the `self.timeframe` attribute. ### Metadata dict @@ -310,12 +312,17 @@ The name of the variable can be chosen at will, but should be prefixed with `cus class Awesomestrategy(IStrategy): # Create custom dictionary cust_info = {} + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: # Check if the entry already exists + if not metadata["pair"] in self._cust_info: + # Create empty entry for this pair + self._cust_info[metadata["pair"]] = {} + if "crosstime" in self.cust_info[metadata["pair"]: - self.cust_info[metadata["pair"]["crosstime"] += 1 + self.cust_info[metadata["pair"]]["crosstime"] += 1 else: - self.cust_info[metadata["pair"]["crosstime"] = 1 + self.cust_info[metadata["pair"]]["crosstime"] = 1 ``` !!! Warning @@ -326,15 +333,15 @@ class Awesomestrategy(IStrategy): *** -### Additional data (informative_pairs) +## Additional data (informative_pairs) -#### Get data for non-tradeable pairs +### Get data for non-tradeable pairs Data for additional, informative pairs (reference pairs) can be beneficial for some strategies. -Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below). +OHLCV data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below). These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting. -The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument. +The pairs need to be specified as tuples in the format `("pair", "timeframe")`, with pair as the first and timeframe as the second argument. Sample: @@ -345,15 +352,17 @@ def informative_pairs(self): ] ``` +A full sample can be found [in the DataProvider section](#complete-data-provider-sample). + !!! Warning As these pairs will be refreshed as part of the regular whitelist refresh, it's best to keep this list short. - All intervals and all pairs can be specified as long as they are available (and active) on the used exchange. - It is however better to use resampling to longer time-intervals when possible + All timeframes and all pairs can be specified as long as they are available (and active) on the used exchange. + It is however better to use resampling to longer timeframes whenever possible to avoid hammering the exchange with too many requests and risk being blocked. *** -### Additional data (DataProvider) +## Additional data (DataProvider) The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. @@ -361,11 +370,16 @@ All methods return `None` in case of failure (do not raise an exception). Please always check the mode of operation to select the correct method to get data (samples see below). -#### Possible options for DataProvider +!!! Warning "Hyperopt" + Dataprovider is available during hyperopt, however it can only be used in `populate_indicators()` within a strategy. + It is not available in `populate_buy()` and `populate_sell()` methods, nor in `populate_indicators()`, if this method located in the hyperopt file. -- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval). -- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist) +### Possible options for DataProvider + +- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their timeframe (pair, timeframe). +- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (i.e. VolumePairlist) - [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). +- [`get_analyzed_dataframe(pair, timeframe)`](#get_analyzed_dataframepair-timeframe) - Returns the analyzed dataframe (after calling `populate_indicators()`, `populate_buy()`, `populate_sell()`) and the time of the latest analysis. - `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. - `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on the Market data structure. - `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. @@ -373,9 +387,9 @@ Please always check the mode of operation to select the correct method to get da - [`ticker(pair)`](#tickerpair) - Returns current ticker data for the pair. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#price-tickers) for more details on the Ticker data structure. - `runmode` - Property containing the current runmode. -#### Example Usages: +### Example Usages -#### *available_pairs* +### *available_pairs* ``` python if self.dp: @@ -383,44 +397,31 @@ if self.dp: print(f"available {pair}, {timeframe}") ``` -#### *current_whitelist()* +### *current_whitelist()* + Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. The strategy might look something like this: -*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day ATR to buy and sell.* +*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day RSI to buy and sell.* -Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least! +Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day RSI. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least! Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use. This is where calling `self.dp.current_whitelist()` comes in handy. ```python -class SampleStrategy(IStrategy): - # strategy init stuff... - - ticker_interval = '5m' - - # more strategy init stuff.. - def informative_pairs(self): - # get access to all pairs available in whitelist. + # get access to all pairs available in whitelist. pairs = self.dp.current_whitelist() # Assign tf to each pair so they can be downloaded and cached for strategy. informative_pairs = [(pair, '1d') for pair in pairs] - return informative_pairs - - def populate_indicators(self, dataframe, metadata): - # Get the informative pair - informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d') - # Get the 14 day ATR. - atr = ta.ATR(informative, timeperiod=14) - # Do other stuff + return informative_pairs ``` -#### *get_pair_dataframe(pair, timeframe)* +### *get_pair_dataframe(pair, timeframe)* ``` python # fetch live / historical candle (OHLCV) data for the first informative pair @@ -431,14 +432,27 @@ if self.dp: ``` !!! Warning "Warning about backtesting" - Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` + Be careful when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` for the backtesting runmode) provides the full time-range in one go, - so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). + so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode. -!!! Warning "Warning in hyperopt" - This option cannot currently be used during hyperopt. +### *get_analyzed_dataframe(pair, timeframe)* -#### *orderbook(pair, maximum)* +This method is used by freqtrade internally to determine the last signal. +It can also be used in specific callbacks to get the signal that caused the action (see [Advanced Strategy Documentation](strategy-advanced.md) for more details on available callbacks). + +``` python +# fetch current dataframe +if self.dp: + dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=metadata['pair'], + timeframe=self.timeframe) +``` + +!!! Note "No data available" + Returns an empty dataframe if the requested pair was not cached. + This should not happen when using whitelisted pairs. + +### *orderbook(pair, maximum)* ``` python if self.dp: @@ -449,10 +463,9 @@ if self.dp: ``` !!! Warning - The order book is not part of the historic data which means backtesting and hyperopt will not work if this - method is used. + The order book is not part of the historic data which means backtesting and hyperopt will not work correctly if this method is used. -#### *ticker(pair)* +### *ticker(pair)* ``` python if self.dp: @@ -469,8 +482,138 @@ if self.dp: does not always fills in the `last` field (so it can be None), etc. So you need to carefully verify the ticker data returned from the exchange and add appropriate error handling / defaults. +!!! Warning "Warning about backtesting" + This method will always return up-to-date values - so usage during backtesting / hyperopt will lead to wrong results. + +### Complete Data-provider sample + +```python +from freqtrade.strategy import IStrategy, merge_informative_pair +from pandas import DataFrame + +class SampleStrategy(IStrategy): + # strategy init stuff... + + timeframe = '5m' + + # more strategy init stuff.. + + def informative_pairs(self): + + # get access to all pairs available in whitelist. + pairs = self.dp.current_whitelist() + # Assign tf to each pair so they can be downloaded and cached for strategy. + informative_pairs = [(pair, '1d') for pair in pairs] + # Optionally Add additional "static" pairs + informative_pairs += [("ETH/USDT", "5m"), + ("BTC/TUSD", "15m"), + ] + return informative_pairs + + def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + if not self.dp: + # Don't do anything if DataProvider is not available. + return dataframe + + inf_tf = '1d' + # Get the informative pair + informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe=inf_tf) + # Get the 14 day rsi + informative['rsi'] = ta.RSI(informative, timeperiod=14) + + # Use the helper function merge_informative_pair to safely merge the pair + # Automatically renames the columns and merges a shorter timeframe dataframe and a longer timeframe informative pair + # use ffill to have the 1d value available in every row throughout the day. + # Without this, comparisons between columns of the original and the informative pair would only work once per day. + # Full documentation of this method, see below + dataframe = merge_informative_pair(dataframe, informative, self.timeframe, inf_tf, ffill=True) + + # Calculate rsi of the original dataframe (5m timeframe) + dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14) + + # Do other stuff + # ... + + return dataframe + + def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: + + dataframe.loc[ + ( + (qtpylib.crossed_above(dataframe['rsi'], 30)) & # Signal: RSI crosses above 30 + (dataframe['rsi_1d'] < 30) & # Ensure daily RSI is < 30 + (dataframe['volume'] > 0) # Ensure this candle had volume (important for backtesting) + ), + 'buy'] = 1 + +``` + *** -### Additional data (Wallets) + +## Helper functions + +### *merge_informative_pair()* + +This method helps you merge an informative pair to a regular dataframe without lookahead bias. +It's there to help you merge the dataframe in a safe and consistent way. + +Options: + +- Rename the columns for you to create unique columns +- Merge the dataframe without lookahead bias +- Forward-fill (optional) + +All columns of the informative dataframe will be available on the returning dataframe in a renamed fashion: + +!!! Example "Column renaming" + Assuming `inf_tf = '1d'` the resulting columns will be: + + ``` python + 'date', 'open', 'high', 'low', 'close', 'rsi' # from the original dataframe + 'date_1d', 'open_1d', 'high_1d', 'low_1d', 'close_1d', 'rsi_1d' # from the informative dataframe + ``` + +??? Example "Column renaming - 1h" + Assuming `inf_tf = '1h'` the resulting columns will be: + + ``` python + 'date', 'open', 'high', 'low', 'close', 'rsi' # from the original dataframe + 'date_1h', 'open_1h', 'high_1h', 'low_1h', 'close_1h', 'rsi_1h' # from the informative dataframe + ``` + +??? Example "Custom implementation" + A custom implementation for this is possible, and can be done as follows: + + ``` python + + # Shift date by 1 candle + # This is necessary since the data is always the "open date" + # and a 15m candle starting at 12:15 should not know the close of the 1h candle from 12:00 to 13:00 + minutes = timeframe_to_minutes(inf_tf) + # Only do this if the timeframes are different: + informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes, 'm') + + # Rename columns to be unique + informative.columns = [f"{col}_{inf_tf}" for col in informative.columns] + # Assuming inf_tf = '1d' - then the columns will now be: + # date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d + + # Combine the 2 dataframes + # all indicators on the informative sample MUST be calculated before this point + dataframe = pd.merge(dataframe, informative, left_on='date', right_on=f'date_merge_{inf_tf}', how='left') + # FFill to have the 1d value available in every row throughout the day. + # Without this, comparisons would only work once per day. + dataframe = dataframe.ffill() + + ``` + +!!! Warning "Informative timeframe < timeframe" + Using informative timeframes smaller than the dataframe timeframe is not recommended with this method, as it will not use any of the additional information this would provide. + To use the more detailed information properly, more advanced methods should be applied (which are out of scope for freqtrade documentation, as it'll depend on the respective need). + +*** + +## Additional data (Wallets) The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. @@ -486,14 +629,15 @@ if self.wallets: total_eth = self.wallets.get_total('ETH') ``` -#### Possible options for Wallets +### Possible options for Wallets - `get_free(asset)` - currently available balance to trade - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above *** -### Additional data (Trades) + +## Additional data (Trades) A history of Trades can be retrieved in the strategy by querying the database. @@ -539,30 +683,30 @@ Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5% (ratio of !!! Warning Trade history is not available during backtesting or hyperopt. -### Prevent trades from happening for a specific pair +## Prevent trades from happening for a specific pair Freqtrade locks pairs automatically for the current candle (until that candle is over) when a pair is sold, preventing an immediate re-buy of that pair. Locked pairs will show the message `Pair is currently locked.`. -#### Locking pairs from within the strategy +### Locking pairs from within the strategy Sometimes it may be desired to lock a pair after certain events happen (e.g. multiple losing trades in a row). -Freqtrade has an easy method to do this from within the strategy, by calling `self.lock_pair(pair, until)`. -`until` must be a datetime object in the future, after which trading will be reenabled for that pair. +Freqtrade has an easy method to do this from within the strategy, by calling `self.lock_pair(pair, until, [reason])`. +`until` must be a datetime object in the future, after which trading will be re-enabled for that pair, while `reason` is an optional string detailing why the pair was locked. Locks can also be lifted manually, by calling `self.unlock_pair(pair)`. To verify if a pair is currently locked, use `self.is_pair_locked(pair)`. !!! Note - Locked pairs are not persisted, so a restart of the bot, or calling `/reload_conf` will reset locked pairs. + Locked pairs will always be rounded up to the next candle. So assuming a `5m` timeframe, a lock with `until` set to 10:18 will lock the pair until the candle from 10:15-10:20 will be finished. !!! Warning - Locking pairs is not functioning during backtesting. + Locking pairs is not available during backtesting. -##### Pair locking example +#### Pair locking example ``` python from freqtrade.persistence import Trade @@ -584,7 +728,7 @@ if self.config['runmode'].value in ('live', 'dry_run'): self.lock_pair(metadata['pair'], until=datetime.now(timezone.utc) + timedelta(hours=12)) ``` -### Print created dataframe +## Print created dataframe To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`. You may also want to print the pair so it's clear what data is currently shown. @@ -608,36 +752,7 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: Printing more than a few rows is also possible (simply use `print(dataframe)` instead of `print(dataframe.tail())`), however not recommended, as that will be very verbose (~500 lines per pair every 5 seconds). -### Specify custom strategy location - -If you want to use a strategy from a different directory you can pass `--strategy-path` - -```bash -freqtrade trade --strategy AwesomeStrategy --strategy-path /some/directory -``` - -### Derived strategies - -The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched: - -``` python -class MyAwesomeStrategy(IStrategy): - ... - stoploss = 0.13 - trailing_stop = False - # All other attributes and methods are here as they - # should be in any custom strategy... - ... - -class MyAwesomeStrategy2(MyAwesomeStrategy): - # Override something - stoploss = 0.08 - trailing_stop = True -``` - -Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need. - -### Common mistakes when developing strategies +## Common mistakes when developing strategies Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future. This is a common pain-point, which can cause huge differences between backtesting and dry/live run methods, since they all use data which is not available during dry/live runs, so these strategies will perform well during backtesting, but will fail / perform badly in real conditions. @@ -649,14 +764,12 @@ The following lists some common patterns which should be avoided to prevent frus - don't use `dataframe['volume'].mean()`. This uses the full DataFrame for backtesting, including data from the future. Use `dataframe['volume'].rolling().mean()` instead - don't use `.resample('1h')`. This uses the left border of the interval, so moves data from an hour to the start of the hour. Use `.resample('1h', label='right')` instead. -### Further strategy ideas +## Further strategy ideas To get additional Ideas for strategies, head over to our [strategy repository](https://github.com/freqtrade/freqtrade-strategies). Feel free to use them as they are - but results will depend on the current market situation, pairs used etc. - therefore please backtest the strategy for your exchange/desired pairs first, evaluate carefully, use at your own risk. Feel free to use any of them as inspiration for your own strategies. We're happy to accept Pull Requests containing new Strategies to that repo. -We also got a *strategy-sharing* channel in our [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) which is a great place to get and/or share ideas. - ## Next step Now you have a perfect strategy you probably want to backtest it. diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index d26d684ce..90e39fd76 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -18,7 +18,7 @@ config = Configuration.from_files([]) # config = Configuration.from_files(["config.json"]) # Define some constants -config["ticker_interval"] = "5m" +config["timeframe"] = "5m" # Name of the strategy class config["strategy"] = "SampleStrategy" # Location of the data @@ -33,7 +33,7 @@ pair = "BTC_USDT" from freqtrade.data.history import load_pair_history candles = load_pair_history(datadir=data_location, - timeframe=config["ticker_interval"], + timeframe=config["timeframe"], pair=pair) # Confirm success @@ -85,10 +85,44 @@ Analyze a trades dataframe (also used below for plotting) ```python -from freqtrade.data.btanalysis import load_backtest_data +from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats -# Load backtest results -trades = load_backtest_data(config["user_data_dir"] / "backtest_results/backtest-result.json") +# if backtest_dir points to a directory, it'll automatically load the last backtest file. +backtest_dir = config["user_data_dir"] / "backtest_results" +# backtest_dir can also point to a specific file +# backtest_dir = config["user_data_dir"] / "backtest_results/backtest-result-2020-07-01_20-04-22.json" +``` + + +```python +# You can get the full backtest statistics by using the following command. +# This contains all information used to generate the backtest result. +stats = load_backtest_stats(backtest_dir) + +strategy = 'SampleStrategy' +# All statistics are available per strategy, so if `--strategy-list` was used during backtest, this will be reflected here as well. +# Example usages: +print(stats['strategy'][strategy]['results_per_pair']) +# Get pairlist used for this backtest +print(stats['strategy'][strategy]['pairlist']) +# Get market change (average change of all pairs from start to end of the backtest period) +print(stats['strategy'][strategy]['market_change']) +# Maximum drawdown () +print(stats['strategy'][strategy]['max_drawdown']) +# Maximum drawdown start and end +print(stats['strategy'][strategy]['drawdown_start']) +print(stats['strategy'][strategy]['drawdown_end']) + + +# Get strategy comparison (only relevant if multiple strategies were compared) +print(stats['strategy_comparison']) + +``` + + +```python +# Load backtested trades as dataframe +trades = load_backtest_data(backtest_dir) # Show value-counts per pair trades.groupby("pair")["sell_reason"].value_counts() diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index f683ae8da..40481684d 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -9,7 +9,7 @@ Telegram user id. Start a chat with the [Telegram BotFather](https://telegram.me/BotFather) -Send the message `/newbot`. +Send the message `/newbot`. *BotFather response:* @@ -35,40 +35,126 @@ Copy the API Token (`22222222:APITOKEN` in the above example) and keep use it fo Don't forget to start the conversation with your bot, by clicking `/START` button -### 2. Get your user id +### 2. Telegram user_id + +#### Get your user id Talk to the [userinfobot](https://telegram.me/userinfobot) Get your "Id", you will use it for the config parameter `chat_id`. +#### Use Group id + +You can use bots in telegram groups by just adding them to the group. You can find the group id by first adding a [RawDataBot](https://telegram.me/rawdatabot) to your group. The Group id is shown as id in the `"chat"` section, which the RawDataBot will send to you: + +``` json +"chat":{ + "id":-1001332619709 +} +``` + +For the Freqtrade configuration, you can then use the the full value (including `-` if it's there) as string: + +```json + "chat_id": "-1001332619709" +``` + +## Control telegram noise + +Freqtrade provides means to control the verbosity of your telegram bot. +Each setting has the following possible values: + +* `on` - Messages will be sent, and user will be notified. +* `silent` - Message will be sent, Notification will be without sound / vibration. +* `off` - Skip sending a message-type all together. + +Example configuration showing the different settings: + +``` json +"telegram": { + "enabled": true, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id", + "notification_settings": { + "status": "silent", + "warning": "on", + "startup": "off", + "buy": "silent", + "sell": "on", + "buy_cancel": "silent", + "sell_cancel": "on" + } + }, +``` + +## Create a custom keyboard (command shortcut buttons) + +Telegram allows us to create a custom keyboard with buttons for commands. +The default custom keyboard looks like this. + +```python +[ + ["/daily", "/profit", "/balance"], # row 1, 3 commands + ["/status", "/status table", "/performance"], # row 2, 3 commands + ["/count", "/start", "/stop", "/help"] # row 3, 4 commands +] +``` + +### Usage + +You can create your own keyboard in `config.json`: + +``` json +"telegram": { + "enabled": true, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id", + "keyboard": [ + ["/daily", "/stats", "/balance", "/profit"], + ["/status table", "/performance"], + ["/reload_config", "/count", "/logs"] + ] + }, +``` + +!!! Note "Supported Commands" + Only the following commands are allowed. Command arguments are not supported! + + `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopbuy`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version` + ## Telegram commands Per default, the Telegram bot shows predefined commands. Some commands are only available by sending them to the bot. The table below list the official commands. You can ask at any moment for help with `/help`. -| Command | Default | Description | -|----------|---------|-------------| -| `/start` | | Starts the trader -| `/stop` | | Stops the trader -| `/stopbuy` | | Stops the trader from opening new trades. Gracefully closes open trades according to their rules. -| `/reload_conf` | | Reloads the configuration file -| `/show_config` | | Shows part of the current configuration with relevant settings to operation -| `/status` | | Lists all open trades -| `/status table` | | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**) -| `/count` | | Displays number of trades used and available -| `/profit` | | Display a summary of your profit/loss from close trades and some stats about your performance -| `/forcesell ` | | Instantly sells the given trade (Ignoring `minimum_roi`). -| `/forcesell all` | | Instantly sells all open trades (Ignoring `minimum_roi`). -| `/forcebuy [rate]` | | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) -| `/performance` | | Show performance of each finished trade grouped by pair -| `/balance` | | Show account balance per currency -| `/daily ` | 7 | Shows profit or loss per day, over the last n days -| `/whitelist` | | Show the current whitelist -| `/blacklist [pair]` | | Show the current blacklist, or adds a pair to the blacklist. -| `/edge` | | Show validated pairs by Edge if it is enabled. -| `/help` | | Show help message -| `/version` | | Show version +| Command | Description | +|----------|-------------| +| `/start` | Starts the trader +| `/stop` | Stops the trader +| `/stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules. +| `/reload_config` | Reloads the configuration file +| `/show_config` | Shows part of the current configuration with relevant settings to operation +| `/logs [limit]` | Show last log messages. +| `/status` | Lists all open trades +| `/status table` | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**) +| `/trades [limit]` | List all recently closed trades in a table format. +| `/delete ` | Delete a specific trade from the Database. Tries to close open orders. Requires manual handling of this trade on the exchange. +| `/count` | Displays number of trades used and available +| `/locks` | Show currently locked pairs. +| `/profit` | Display a summary of your profit/loss from close trades and some stats about your performance +| `/forcesell ` | Instantly sells the given trade (Ignoring `minimum_roi`). +| `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`). +| `/forcebuy [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) +| `/performance` | Show performance of each finished trade grouped by pair +| `/balance` | Show account balance per currency +| `/daily ` | Shows profit or loss per day, over the last n days (n defaults to 7) +| `/stats` | Shows Wins / losses by Sell reason as well as Avg. holding durations for buys and sells +| `/whitelist` | Show the current whitelist +| `/blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist. +| `/edge` | Show validated pairs by Edge if it is enabled. +| `/help` | Show help message +| `/version` | Show version ## Telegram commands in action @@ -85,14 +171,14 @@ Below, example of Telegram message you will receive for each command. ### /stopbuy -> **status:** `Setting max_open_trades to 0. Run /reload_conf to reset.` +> **status:** `Setting max_open_trades to 0. Run /reload_config to reset.` Prevents the bot from opening new trades by temporarily setting "max_open_trades" to 0. Open trades will be handled via their regular rules (ROI / Sell-signal, stoploss, ...). After this, give the bot time to close off open trades (can be checked via `/status table`). Once all positions are sold, run `/stop` to completely stop the bot. -`/reload_conf` resets "max_open_trades" to the value set in the configuration and resets this command. +`/reload_config` resets "max_open_trades" to the value set in the configuration and resets this command. !!! Warning The stop-buy signal is ONLY active while the bot is running, and is not persisted anyway, so restarting the bot will cause this to reset. @@ -113,6 +199,7 @@ For each open trade, the bot will send you the following message. ### /status table Return the status of all open trades in a table format. + ``` ID Pair Since Profit ---- -------- ------- -------- @@ -123,6 +210,7 @@ Return the status of all open trades in a table format. ### /count Return the number of trades used and available. + ``` current max --------- ----- @@ -156,7 +244,7 @@ Return a summary of your profit/loss and performance. Note that for this to work, `forcebuy_enable` needs to be set to true. -[More details](configuration.md/#understand-forcebuy_enable) +[More details](configuration.md#understand-forcebuy_enable) ### /performance @@ -208,15 +296,15 @@ Shows the current whitelist Shows the current blacklist. If Pair is set, then this pair will be added to the pairlist. -Also supports multiple pairs, seperated by a space. -Use `/reload_conf` to reset the blacklist. +Also supports multiple pairs, separated by a space. +Use `/reload_config` to reset the blacklist. > Using blacklist `StaticPairList` with 2 pairs >`DODGE/BTC`, `HOT/BTC`. ### /edge -Shows pairs validated by Edge along with their corresponding winrate, expectancy and stoploss values. +Shows pairs validated by Edge along with their corresponding win-rate, expectancy and stoploss values. > **Edge only validated following pairs:** ``` diff --git a/docs/updating.md b/docs/updating.md new file mode 100644 index 000000000..b23ce32dc --- /dev/null +++ b/docs/updating.md @@ -0,0 +1,31 @@ +# How to update + +To update your freqtrade installation, please use one of the below methods, corresponding to your installation method. + +## docker-compose + +!!! Note "Legacy installations using the `master` image" + We're switching from master to stable for the release Images - please adjust your docker-file and replace `freqtradeorg/freqtrade:master` with `freqtradeorg/freqtrade:stable` + +``` bash +docker-compose pull +docker-compose up -d +``` + +## Installation via setup script + +``` bash +./setup.sh --update +``` + +!!! Note + Make sure to run this command with your virtual environment disabled! + +## Plain native installation + +Please ensure that you're also updating dependencies - otherwise things might break without you noticing. + +``` bash +git pull +pip install -U -r requirements.txt +``` diff --git a/docs/utils.md b/docs/utils.md index 7ed31376f..409bcc134 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -62,7 +62,7 @@ $ freqtrade new-config --config config_binance.json ? Please insert your stake currency: BTC ? Please insert your stake amount: 0.05 ? Please insert max_open_trades (Integer or 'unlimited'): 3 -? Please insert your timeframe (ticker interval): 5m +? Please insert your desired timeframe (e.g. 5m): 5m ? Please insert your display Currency (for reporting): USD ? Select exchange binance ? Do you want to enable Telegram? No @@ -423,7 +423,7 @@ freqtrade test-pairlist --config config.json --quote USDT BTC ## List Hyperopt results -You can list the hyperoptimization epochs the Hyperopt module evaluated previously with the `hyperopt-list` subcommand. +You can list the hyperoptimization epochs the Hyperopt module evaluated previously with the `hyperopt-list` sub-command. ``` usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] @@ -433,9 +433,10 @@ usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--max-avg-time FLOAT] [--min-avg-profit FLOAT] [--max-avg-profit FLOAT] [--min-total-profit FLOAT] - [--max-total-profit FLOAT] [--no-color] - [--print-json] [--no-details] - [--export-csv FILE] + [--max-total-profit FLOAT] + [--min-objective FLOAT] [--max-objective FLOAT] + [--no-color] [--print-json] [--no-details] + [--hyperopt-filename PATH] [--export-csv FILE] optional arguments: -h, --help show this help message and exit @@ -443,20 +444,27 @@ optional arguments: --profitable Select only profitable epochs. --min-trades INT Select epochs with more than INT trades. --max-trades INT Select epochs with less than INT trades. - --min-avg-time FLOAT Select epochs on above average time. - --max-avg-time FLOAT Select epochs on under average time. + --min-avg-time FLOAT Select epochs above average time. + --max-avg-time FLOAT Select epochs below average time. --min-avg-profit FLOAT - Select epochs on above average profit. + Select epochs above average profit. --max-avg-profit FLOAT - Select epochs on below average profit. + Select epochs below average profit. --min-total-profit FLOAT - Select epochs on above total profit. + Select epochs above total profit. --max-total-profit FLOAT - Select epochs on below total profit. + Select epochs below total profit. + --min-objective FLOAT + Select epochs above objective. + --max-objective FLOAT + Select epochs below objective. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. - --print-json Print best result detailization in JSON format. + --print-json Print output in JSON format. --no-details Do not print best epoch details. + --hyperopt-filename FILENAME + Hyperopt result filename.Example: `--hyperopt- + filename=hyperopt_results_2020-09-27_16-20-48.pickle` --export-csv FILE Export to CSV-File. This will disable table print. Example: --export-csv hyperopt.csv @@ -476,7 +484,11 @@ Common arguments: --userdir PATH, --user-data-dir PATH Path to userdata directory. ``` - + +!!! Note + `hyperopt-list` will automatically use the latest available hyperopt results file. + You can override this using the `--hyperopt-filename` argument, and specify another, available filename (without path!). + ### Examples List all results, print details of the best result at the end: @@ -497,17 +509,41 @@ You can show the details of any hyperoptimization epoch previously evaluated by usage: freqtrade hyperopt-show [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--best] [--profitable] [-n INT] [--print-json] - [--no-header] + [--hyperopt-filename PATH] [--no-header] optional arguments: -h, --help show this help message and exit --best Select only best epochs. --profitable Select only profitable epochs. -n INT, --index INT Specify the index of the epoch to print details for. - --print-json Print best result detailization in JSON format. + --print-json Print output in JSON format. + --hyperopt-filename FILENAME + Hyperopt result filename.Example: `--hyperopt- + filename=hyperopt_results_2020-09-27_16-20-48.pickle` --no-header Do not print epoch details header. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + ``` +!!! Note + `hyperopt-show` will automatically use the latest available hyperopt results file. + You can override this using the `--hyperopt-filename` argument, and specify another, available filename (without path!). + ### Examples Print details for the epoch 168 (the number of the epoch is shown by the `hyperopt-list` subcommand or by Hyperopt itself during hyperoptimization run): diff --git a/docs/webhook-config.md b/docs/webhook-config.md index 70a41dd46..db6d4d1ef 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -47,6 +47,7 @@ Different payloads can be configured for different events. Not all fields are ne The fields in `webhook.webhookbuy` are filled when the bot executes a buy. Parameters are filled using string.format. Possible parameters are: +* `trade_id` * `exchange` * `pair` * `limit` @@ -63,6 +64,7 @@ Possible parameters are: The fields in `webhook.webhookbuycancel` are filled when the bot cancels a buy order. Parameters are filled using string.format. Possible parameters are: +* `trade_id` * `exchange` * `pair` * `limit` @@ -79,6 +81,7 @@ Possible parameters are: The fields in `webhook.webhooksell` are filled when the bot sells a trade. Parameters are filled using string.format. Possible parameters are: +* `trade_id` * `exchange` * `pair` * `gain` @@ -100,6 +103,7 @@ Possible parameters are: The fields in `webhook.webhooksellcancel` are filled when the bot cancels a sell order. Parameters are filled using string.format. Possible parameters are: +* `trade_id` * `exchange` * `pair` * `gain` diff --git a/docs/windows_installation.md b/docs/windows_installation.md new file mode 100644 index 000000000..5341ce96b --- /dev/null +++ b/docs/windows_installation.md @@ -0,0 +1,57 @@ +We **strongly** recommend that Windows users use [Docker](docker.md) as this will work much easier and smoother (also more secure). + +If that is not possible, try using the Windows Linux subsystem (WSL) - for which the Ubuntu instructions should work. +Otherwise, try the instructions below. + +## Install freqtrade manually + +!!! Note + Make sure to use 64bit Windows and 64bit Python to avoid problems with backtesting or hyperopt due to the memory constraints 32bit applications have under Windows. + +!!! Hint + Using the [Anaconda Distribution](https://www.anaconda.com/distribution/) under Windows can greatly help with installation problems. Check out the [Anaconda installation section](installation.md#Anaconda) in this document for more information. + +### 1. Clone the git repository + +```bash +git clone https://github.com/freqtrade/freqtrade.git +``` + +### 2. Install ta-lib + +Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). + +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.19‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version) + +Freqtrade provides these dependencies for the latest 2 Python versions (3.7 and 3.8) and for 64bit Windows. +Other versions must be downloaded from the above link. + +``` powershell +cd \path\freqtrade +python -m venv .env +.env\Scripts\activate.ps1 +# optionally install ta-lib from wheel +# Eventually adjust the below filename to match the downloaded wheel +pip install build_helpers/TA_Lib-0.4.19-cp38-cp38-win_amd64.whl +pip install -r requirements.txt +pip install -e . +freqtrade +``` + +!!! Note "Use Powershell" + The above installation script assumes you're using powershell on a 64bit windows. + Commands for the legacy CMD windows console may differ. + +> Thanks [Owdr](https://github.com/Owdr) for the commands. Source: [Issue #222](https://github.com/freqtrade/freqtrade/issues/222) + +### Error during installation on Windows + +``` bash +error: Microsoft Visual C++ 14.0 is required. Get it with "Microsoft Visual C++ Build Tools": http://landinghub.visualstudio.com/visual-cpp-build-tools +``` + +Unfortunately, many packages requiring compilation don't provide a pre-built wheel. It is therefore mandatory to have a C/C++ compiler installed and available for your python environment to use. + +The easiest way is to download install Microsoft Visual Studio Community [here](https://visualstudio.microsoft.com/downloads/) and make sure to install "Common Tools for Visual C++" to enable building C code on Windows. Unfortunately, this is a heavy download / dependency (~4Gb) so you might want to consider WSL or [docker](docker.md) first. + +--- diff --git a/environment.yml b/environment.yml index 86ea03519..746c4b912 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: - conda-forge dependencies: # Required for app - - python>=3.6 + - python>=3.7 - pip - wheel - numpy diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py index 97ed9ae67..ab4c7a110 100644 --- a/freqtrade/__main__.py +++ b/freqtrade/__main__.py @@ -3,10 +3,11 @@ __main__.py for Freqtrade To launch Freqtrade as a module -> python -m freqtrade (with Python >= 3.6) +> python -m freqtrade (with Python >= 3.7) """ from freqtrade import main + if __name__ == '__main__': main.main() diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 2d0c7733c..21c5d6812 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -8,22 +8,15 @@ Note: Be careful with file-scoped imports in these subfiles. """ from freqtrade.commands.arguments import Arguments from freqtrade.commands.build_config_commands import start_new_config -from freqtrade.commands.data_commands import (start_convert_data, - start_download_data) -from freqtrade.commands.deploy_commands import (start_create_userdir, - start_new_hyperopt, +from freqtrade.commands.data_commands import (start_convert_data, start_download_data, + start_list_data) +from freqtrade.commands.deploy_commands import (start_create_userdir, start_new_hyperopt, start_new_strategy) -from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, - start_hyperopt_show) -from freqtrade.commands.list_commands import (start_list_exchanges, - start_list_hyperopts, - start_list_markets, - start_list_strategies, - start_list_timeframes, - start_show_trades) -from freqtrade.commands.optimize_commands import (start_backtesting, - start_edge, start_hyperopt) +from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show +from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, + start_list_markets, start_list_strategies, + start_list_timeframes, start_show_trades) +from freqtrade.commands.optimize_commands import start_backtesting, start_edge, start_hyperopt from freqtrade.commands.pairlist_commands import start_test_pairlist -from freqtrade.commands.plot_commands import (start_plot_dataframe, - start_plot_profit) +from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit from freqtrade.commands.trade_commands import start_trading diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 1b7bbfeb5..a6c8a245f 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -9,24 +9,27 @@ from typing import Any, Dict, List, Optional from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS from freqtrade.constants import DEFAULT_CONFIG + ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] ARGS_STRATEGY = ["strategy", "strategy_path"] ARGS_TRADE = ["db_url", "sd_notify", "dry_run"] -ARGS_COMMON_OPTIMIZE = ["ticker_interval", "timerange", +ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", "max_open_trades", "stake_amount", "fee"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", + "enable_protections", "strategy_list", "export", "exportfilename"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", - "position_stacking", "epochs", "spaces", - "use_max_market_positions", "print_all", + "position_stacking", "use_max_market_positions", + "enable_protections", + "epochs", "spaces", "print_all", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", - "hyperopt_continue", "hyperopt_loss"] + "hyperopt_loss"] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] @@ -41,7 +44,8 @@ ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"] ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one_column", "print_csv", "base_currencies", "quote_currencies", "list_pairs_all"] -ARGS_TEST_PAIRLIST = ["config", "quote_currencies", "print_one_column", "list_pairs_print_json"] +ARGS_TEST_PAIRLIST = ["verbosity", "config", "quote_currencies", "print_one_column", + "list_pairs_print_json"] ARGS_CREATE_USERDIR = ["user_data_dir", "reset"] @@ -54,15 +58,17 @@ ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] -ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", +ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] + +ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", - "timerange", "ticker_interval", "no_trades"] + "timerange", "timeframe", "no_trades"] ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", - "trade_source", "ticker_interval"] + "trade_source", "timeframe"] ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"] @@ -71,14 +77,15 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit", "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit", + "hyperopt_list_min_objective", "hyperopt_list_max_objective", "print_colorized", "print_json", "hyperopt_list_no_details", - "export_csv"] + "hyperoptexportfilename", "export_csv"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", - "print_json", "hyperopt_show_no_header"] + "print_json", "hyperoptexportfilename", "hyperopt_show_no_header"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", - "list-markets", "list-pairs", "list-strategies", + "list-markets", "list-pairs", "list-strategies", "list-data", "list-hyperopts", "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit", "show-trades"] @@ -158,16 +165,14 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.commands import (start_create_userdir, start_convert_data, - start_download_data, - start_hyperopt_list, start_hyperopt_show, + from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir, + start_download_data, start_edge, start_hyperopt, + start_hyperopt_list, start_hyperopt_show, start_list_data, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, - start_list_timeframes, start_new_config, - start_new_hyperopt, start_new_strategy, - start_plot_dataframe, start_plot_profit, start_show_trades, - start_backtesting, start_hyperopt, start_edge, - start_test_pairlist, start_trading) + start_list_timeframes, start_new_config, start_new_hyperopt, + start_new_strategy, start_plot_dataframe, start_plot_profit, + start_show_trades, start_test_pairlist, start_trading) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -233,6 +238,15 @@ class Arguments: convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False)) self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd) + # Add list-data subcommand + list_data_cmd = subparsers.add_parser( + 'list-data', + help='List downloaded data.', + parents=[_common_parser], + ) + list_data_cmd.set_defaults(func=start_list_data) + self._build_args(optionlist=ARGS_LIST_DATA, parser=list_data_cmd) + # Add backtesting subcommand backtesting_cmd = subparsers.add_parser('backtesting', help='Backtesting module.', parents=[_common_parser, _strategy_parser]) @@ -318,7 +332,7 @@ class Arguments: # Add list-timeframes subcommand list_timeframes_cmd = subparsers.add_parser( 'list-timeframes', - help='Print available ticker intervals (timeframes) for the exchange.', + help='Print available timeframes for the exchange.', parents=[_common_parser], ) list_timeframes_cmd.set_defaults(func=start_list_timeframes) @@ -354,7 +368,7 @@ class Arguments: plot_profit_cmd = subparsers.add_parser( 'plot-profit', help='Generate plot showing profits.', - parents=[_common_parser], + parents=[_common_parser, _strategy_parser], ) plot_profit_cmd.set_defaults(func=start_plot_profit) self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 87098f53c..7bdbcc057 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -1,13 +1,15 @@ import logging from pathlib import Path -from typing import Any, Dict +from typing import Any, Dict, List from questionary import Separator, prompt from freqtrade.constants import UNLIMITED_STAKE_AMOUNT -from freqtrade.exchange import available_exchanges, MAP_EXCHANGE_CHILDCLASS -from freqtrade.misc import render_template from freqtrade.exceptions import OperationalException +from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges +from freqtrade.misc import render_template + + logger = logging.getLogger(__name__) @@ -46,7 +48,7 @@ def ask_user_config() -> Dict[str, Any]: Interactive questions built using https://github.com/tmbo/questionary :returns: Dict with keys to put into template """ - questions = [ + questions: List[Dict[str, Any]] = [ { "type": "confirm", "name": "dry_run", @@ -75,8 +77,8 @@ def ask_user_config() -> Dict[str, Any]: }, { "type": "text", - "name": "ticker_interval", - "message": "Please insert your timeframe (ticker interval):", + "name": "timeframe", + "message": "Please insert your desired timeframe (e.g. 5m):", "default": "5m", }, { diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index ee9208c33..668b4abf5 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -4,6 +4,7 @@ Definition of cli arguments used in arguments.py from argparse import ArgumentTypeError from freqtrade import __version__, constants +from freqtrade.constants import HYPEROPT_LOSS_BUILTIN def check_int_positive(value: str) -> int: @@ -110,8 +111,8 @@ AVAILABLE_CLI_OPTIONS = { action='store_true', ), # Optimize common - "ticker_interval": Arg( - '-i', '--ticker-interval', + "timeframe": Arg( + '-i', '--timeframe', '--ticker-interval', help='Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`).', ), "timerange": Arg( @@ -143,6 +144,14 @@ AVAILABLE_CLI_OPTIONS = { action='store_false', default=True, ), + "enable_protections": Arg( + '--enable-protections', '--enableprotections', + help='Enable protections for backtesting.' + 'Will slow backtesting down by a considerable amount, but will include ' + 'configured protections', + action='store_true', + default=False, + ), "strategy_list": Arg( '--strategy-list', help='Provide a space-separated list of strategies to backtest. ' @@ -252,23 +261,19 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=1, ), - "hyperopt_continue": Arg( - "--continue", - help="Continue hyperopt from previous runs. " - "By default, temporary files will be removed and hyperopt will start from scratch.", - default=False, - action='store_true', - ), "hyperopt_loss": Arg( '--hyperopt-loss', help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). ' 'Different functions can generate completely different results, ' 'since the target for optimization is different. Built-in Hyperopt-loss-functions are: ' - 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, ' - 'SortinoHyperOptLoss, SortinoHyperOptLossDaily.' - '(default: `%(default)s`).', + f'{", ".join(HYPEROPT_LOSS_BUILTIN)}', metavar='NAME', - default=constants.DEFAULT_HYPEROPT_LOSS, + ), + "hyperoptexportfilename": Arg( + '--hyperopt-filename', + help='Hyperopt result filename.' + 'Example: `--hyperopt-filename=hyperopt_results_2020-09-27_16-20-48.pickle`', + metavar='FILENAME', ), # List exchanges "print_one_column": Arg( @@ -357,13 +362,11 @@ AVAILABLE_CLI_OPTIONS = { '--data-format-ohlcv', help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).', choices=constants.AVAILABLE_DATAHANDLERS, - default='json' ), "dataformat_trades": Arg( '--data-format-trades', help='Storage format for downloaded trades data. (default: `%(default)s`).', choices=constants.AVAILABLE_DATAHANDLERS, - default='jsongz' ), "exchange": Arg( '--exchange', @@ -375,7 +378,7 @@ AVAILABLE_CLI_OPTIONS = { help='Specify which tickers to download. Space-separated list. ' 'Default: `1m 5m`.', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', - '6h', '8h', '12h', '1d', '3d', '1w'], + '6h', '8h', '12h', '1d', '3d', '1w', '2w', '1M', '1y'], default=['1m', '5m'], nargs='+', ), @@ -455,37 +458,49 @@ AVAILABLE_CLI_OPTIONS = { ), "hyperopt_list_min_avg_time": Arg( '--min-avg-time', - help='Select epochs on above average time.', + help='Select epochs above average time.', type=float, metavar='FLOAT', ), "hyperopt_list_max_avg_time": Arg( '--max-avg-time', - help='Select epochs on under average time.', + help='Select epochs below average time.', type=float, metavar='FLOAT', ), "hyperopt_list_min_avg_profit": Arg( '--min-avg-profit', - help='Select epochs on above average profit.', + help='Select epochs above average profit.', type=float, metavar='FLOAT', ), "hyperopt_list_max_avg_profit": Arg( '--max-avg-profit', - help='Select epochs on below average profit.', + help='Select epochs below average profit.', type=float, metavar='FLOAT', ), "hyperopt_list_min_total_profit": Arg( '--min-total-profit', - help='Select epochs on above total profit.', + help='Select epochs above total profit.', type=float, metavar='FLOAT', ), "hyperopt_list_max_total_profit": Arg( '--max-total-profit', - help='Select epochs on below total profit.', + help='Select epochs below total profit.', + type=float, + metavar='FLOAT', + ), + "hyperopt_list_min_objective": Arg( + '--min-objective', + help='Select epochs above objective.', + type=float, + metavar='FLOAT', + ), + "hyperopt_list_max_objective": Arg( + '--max-objective', + help='Select epochs below objective.', type=float, metavar='FLOAT', ), diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index fc3a49f1d..25c7d0436 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -1,19 +1,19 @@ import logging import sys +from collections import defaultdict +from datetime import datetime, timedelta from typing import Any, Dict, List -import arrow - from freqtrade.configuration import TimeRange, setup_utils_configuration -from freqtrade.data.converter import (convert_ohlcv_format, - convert_trades_format) -from freqtrade.data.history import (convert_trades_to_ohlcv, - refresh_backtest_ohlcv_data, +from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format +from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_minutes from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode + logger = logging.getLogger(__name__) @@ -23,18 +23,27 @@ def start_download_data(args: Dict[str, Any]) -> None: """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + if 'days' in config and 'timerange' in config: + raise OperationalException("--days and --timerange are mutually exclusive. " + "You can only specify one or the other.") timerange = TimeRange() if 'days' in config: - time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + time_since = (datetime.now() - timedelta(days=config['days'])).strftime("%Y%m%d") timerange = TimeRange.parse_timerange(f'{time_since}-') + if 'timerange' in config: + timerange = timerange.parse_timerange(config['timerange']) + + # Remove stake-currency to skip checks which are not relevant for datadownload + config['stake_currency'] = '' + if 'pairs' not in config: raise OperationalException( "Downloading data requires a list of pairs. " "Please check the documentation on how to configure this.") - logger.info(f'About to download pairs: {config["pairs"]}, ' - f'intervals: {config["timeframes"]} to {config["datadir"]}') + logger.info(f"About to download pairs: {config['pairs']}, " + f"intervals: {config['timeframes']} to {config['datadir']}") pairs_not_available: List[str] = [] @@ -49,21 +58,21 @@ def start_download_data(args: Dict[str, Any]) -> None: if config.get('download_trades'): pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config["pairs"], datadir=config['datadir'], - timerange=timerange, erase=bool(config.get("erase")), + exchange, pairs=config['pairs'], datadir=config['datadir'], + timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_trades']) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( - pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=bool(config.get("erase")), + pairs=config['pairs'], timeframes=config['timeframes'], + datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], ) else: pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=config["pairs"], timeframes=config["timeframes"], - datadir=config['datadir'], timerange=timerange, erase=bool(config.get("erase")), + exchange, pairs=config['pairs'], timeframes=config['timeframes'], + datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) except KeyboardInterrupt: @@ -88,3 +97,31 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: convert_trades_format(config, convert_from=args['format_from'], convert_to=args['format_to'], erase=args['erase']) + + +def start_list_data(args: Dict[str, Any]) -> None: + """ + List available backtest data + """ + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + from tabulate import tabulate + + from freqtrade.data.history.idatahandler import get_datahandler + dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv']) + + paircombs = dhc.ohlcv_get_available_data(config['datadir']) + + if args['pairs']: + paircombs = [comb for comb in paircombs if comb[0] in args['pairs']] + + print(f"Found {len(paircombs)} pair / timeframe combinations.") + groupedpair = defaultdict(list) + for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))): + groupedpair[pair].append(timeframe) + + if groupedpair: + print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()], + headers=("Pair", "Timeframe"), + tablefmt='psql', stralign='right')) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 86562fa7c..a0105e140 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -4,13 +4,13 @@ from pathlib import Path from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration -from freqtrade.configuration.directory_operations import (copy_sample_files, - create_userdata_dir) +from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template, render_template_with_fallback from freqtrade.state import RunMode + logger = logging.getLogger(__name__) @@ -75,7 +75,7 @@ def start_new_strategy(args: Dict[str, Any]) -> None: if args["strategy"] == "DefaultStrategy": raise OperationalException("DefaultStrategy is not allowed as name.") - new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args["strategy"] + ".py") + new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args['strategy'] + '.py') if new_path.exists(): raise OperationalException(f"`{new_path}` already exists. " @@ -125,15 +125,15 @@ def start_new_hyperopt(args: Dict[str, Any]) -> None: config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - if "hyperopt" in args and args["hyperopt"]: - if args["hyperopt"] == "DefaultHyperopt": + if 'hyperopt' in args and args['hyperopt']: + if args['hyperopt'] == 'DefaultHyperopt': raise OperationalException("DefaultHyperopt is not allowed as name.") - new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py") + new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py') if new_path.exists(): raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Strategy Name.") + "Please choose another Hyperopt Name.") deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) else: raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 517f47d16..fd8f737f0 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -5,9 +5,11 @@ from typing import Any, Dict, List from colorama import init as colorama_init from freqtrade.configuration import setup_utils_configuration +from freqtrade.data.btanalysis import get_latest_hyperopt_file from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode + logger = logging.getLogger(__name__) @@ -35,17 +37,20 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), - 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None) + 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None), + 'filter_min_objective': config.get('hyperopt_list_min_objective', None), + 'filter_max_objective': config.get('hyperopt_list_max_objective', None), } - results_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') + results_file = get_latest_hyperopt_file( + config['user_data_dir'] / 'hyperopt_results', + config.get('hyperoptexportfilename')) # Previous evaluations epochs = Hyperopt.load_previous_results(results_file) total_epochs = len(epochs) - epochs = _hyperopt_filter_epochs(epochs, filteroptions) + epochs = hyperopt_filter_epochs(epochs, filteroptions) if print_colorized: colorama_init(autoreset=True) @@ -78,8 +83,10 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: print_json = config.get('print_json', False) no_header = config.get('hyperopt_show_no_header', False) - results_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') + results_file = get_latest_hyperopt_file( + config['user_data_dir'] / 'hyperopt_results', + config.get('hyperoptexportfilename')) + n = config.get('hyperopt_show_index', -1) filteroptions = { @@ -92,14 +99,16 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), - 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None) + 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None), + 'filter_min_objective': config.get('hyperopt_list_min_objective', None), + 'filter_max_objective': config.get('hyperopt_list_max_objective', None) } # Previous evaluations epochs = Hyperopt.load_previous_results(results_file) total_epochs = len(epochs) - epochs = _hyperopt_filter_epochs(epochs, filteroptions) + epochs = hyperopt_filter_epochs(epochs, filteroptions) filtered_epochs = len(epochs) if n > filtered_epochs: @@ -119,7 +128,7 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: header_str="Epoch details") -def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: +def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: """ Filter our items from the list of hyperopt results """ @@ -127,6 +136,24 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: epochs = [x for x in epochs if x['is_best']] if filteroptions['only_profitable']: epochs = [x for x in epochs if x['results_metrics']['profit'] > 0] + + epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions) + + epochs = _hyperopt_filter_epochs_duration(epochs, filteroptions) + + epochs = _hyperopt_filter_epochs_profit(epochs, filteroptions) + + epochs = _hyperopt_filter_epochs_objective(epochs, filteroptions) + + logger.info(f"{len(epochs)} " + + ("best " if filteroptions['only_best'] else "") + + ("profitable " if filteroptions['only_profitable'] else "") + + "epochs found.") + return epochs + + +def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List: + if filteroptions['filter_min_trades'] > 0: epochs = [ x for x in epochs @@ -137,6 +164,11 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: x for x in epochs if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] ] + return epochs + + +def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List: + if filteroptions['filter_min_avg_time'] is not None: epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] epochs = [ @@ -149,6 +181,12 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: x for x in epochs if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] ] + + return epochs + + +def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List: + if filteroptions['filter_min_avg_profit'] is not None: epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] epochs = [ @@ -173,10 +211,18 @@ def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: x for x in epochs if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] ] + return epochs - logger.info(f"{len(epochs)} " + - ("best " if filteroptions['only_best'] else "") + - ("profitable " if filteroptions['only_profitable'] else "") + - "epochs found.") + +def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List: + + if filteroptions['filter_min_objective'] is not None: + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + + epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']] + if filteroptions['filter_max_objective'] is not None: + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + + epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']] return epochs diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index e5131f9b2..9e6076dfb 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -5,20 +5,20 @@ from collections import OrderedDict from pathlib import Path from typing import Any, Dict, List -from colorama import init as colorama_init -from colorama import Fore, Style import rapidjson +from colorama import Fore, Style +from colorama import init as colorama_init from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException -from freqtrade.exchange import (available_exchanges, ccxt_exchanges, - market_is_active, symbol_is_pair) +from freqtrade.exchange import available_exchanges, ccxt_exchanges, market_is_active from freqtrade.misc import plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode + logger = logging.getLogger(__name__) @@ -102,8 +102,8 @@ def start_list_timeframes(args: Dict[str, Any]) -> None: Print ticker intervals (timeframes) available on Exchange """ config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - # Do not use ticker_interval set in the config - config['ticker_interval'] = None + # Do not use timeframe set in the config + config['timeframe'] = None # Init exchange exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) @@ -163,7 +163,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], 'Base': v['base'], 'Quote': v['quote'], 'Active': market_is_active(v), - **({'Is pair': symbol_is_pair(v['symbol'])} + **({'Is pair': exchange.market_is_tradable(v)} if not pairs_only else {})}) if (args.get('print_one_column', False) or @@ -203,15 +203,16 @@ def start_show_trades(args: Dict[str, Any]) -> None: """ Show trades """ - from freqtrade.persistence import init, Trade import json + + from freqtrade.persistence import Trade, init_db config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if 'db_url' not in config: raise OperationalException("--db-url is required for this command.") logger.info(f'Using DB: "{config["db_url"]}"') - init(config['db_url'], clean_open_orders=False) + init_db(config['db_url'], clean_open_orders=False) tfilter = [] if config.get('trade_ids'): diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 2fc605926..7411ca9c6 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -6,6 +6,7 @@ from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode + logger = logging.getLogger(__name__) @@ -58,6 +59,7 @@ def start_hyperopt(args: Dict[str, Any]) -> None: # Import here to avoid loading hyperopt module when it's not used try: from filelock import FileLock, Timeout + from freqtrade.optimize.hyperopt import Hyperopt except ImportError as e: raise OperationalException( @@ -98,6 +100,7 @@ def start_edge(args: Dict[str, Any]) -> None: :return: None """ from freqtrade.optimize.edge_cli import EdgeCli + # Initialize configuration config = setup_optimize_configuration(args, RunMode.EDGE) logger.info('Starting freqtrade in Edge mode') diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index bf0b217a5..0661cd03c 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -7,6 +7,7 @@ from freqtrade.configuration import setup_utils_configuration from freqtrade.resolvers import ExchangeResolver from freqtrade.state import RunMode + logger = logging.getLogger(__name__) @@ -14,7 +15,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration """ - from freqtrade.pairlist.pairlistmanager import PairListManager + from freqtrade.plugins.pairlistmanager import PairListManager config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) @@ -25,7 +26,6 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: results = {} for curr in quote_currencies: config['stake_currency'] = curr - # Do not use ticker_interval set in the config pairlists = PairListManager(exchange, config) pairlists.refresh_pairlist() results[curr] = pairlists.whitelist diff --git a/freqtrade/commands/trade_commands.py b/freqtrade/commands/trade_commands.py index c058e4f9d..535844844 100644 --- a/freqtrade/commands/trade_commands.py +++ b/freqtrade/commands/trade_commands.py @@ -1,5 +1,4 @@ import logging - from typing import Any, Dict diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index d41ac97ec..607f9cdef 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,7 +1,7 @@ # flake8: noqa: F401 -from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.check_exchange import check_exchange, remove_credentials -from freqtrade.configuration.timerange import TimeRange -from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency +from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.timerange import TimeRange diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 92daaf251..aa36de3ff 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -2,11 +2,11 @@ import logging from typing import Any, Dict from freqtrade.exceptions import OperationalException -from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, - is_exchange_bad, is_exchange_known_ccxt, - is_exchange_officially_supported) +from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, is_exchange_bad, + is_exchange_known_ccxt, is_exchange_officially_supported) from freqtrade.state import RunMode + logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 64f283e42..3b0f778f4 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -1,10 +1,12 @@ import logging from typing import Any, Dict +from freqtrade.state import RunMode + +from .check_exchange import remove_credentials from .config_validation import validate_config_consistency from .configuration import Configuration -from .check_exchange import remove_credentials -from freqtrade.state import RunMode + logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 5ba7ff294..b8829b80f 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -9,6 +9,7 @@ from freqtrade import constants from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode + logger = logging.getLogger(__name__) @@ -73,6 +74,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: _validate_trailing_stoploss(conf) _validate_edge(conf) _validate_whitelist(conf) + _validate_protections(conf) _validate_unlimited_amount(conf) # validate configuration before returning @@ -136,6 +138,10 @@ def _validate_edge(conf: Dict[str, Any]) -> None: "Edge and VolumePairList are incompatible, " "Edge will override whatever pairs VolumePairlist selects." ) + if not conf.get('ask_strategy', {}).get('use_sell_signal', True): + raise OperationalException( + "Edge requires `use_sell_signal` to be True, otherwise no sells will happen." + ) def _validate_whitelist(conf: Dict[str, Any]) -> None: @@ -150,3 +156,22 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: if (pl.get('method') == 'StaticPairList' and not conf.get('exchange', {}).get('pair_whitelist')): raise OperationalException("StaticPairList requires pair_whitelist to be set.") + + +def _validate_protections(conf: Dict[str, Any]) -> None: + """ + Validate protection configuration validity + """ + + for prot in conf.get('protections', []): + if ('stop_duration' in prot and 'stop_duration_candles' in prot): + raise OperationalException( + "Protections must specify either `stop_duration` or `stop_duration_candles`.\n" + f"Please fix the protection {prot.get('method')}" + ) + + if ('lookback_period' in prot and 'lookback_period_candles' in prot): + raise OperationalException( + "Protections must specify either `lookback_period` or `lookback_period_candles`.\n" + f"Please fix the protection {prot.get('method')}" + ) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 7edd9bca1..7bf3e6bf2 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -10,14 +10,14 @@ from typing import Any, Callable, Dict, List, Optional from freqtrade import constants from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings -from freqtrade.configuration.directory_operations import (create_datadir, - create_userdata_dir) +from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir from freqtrade.configuration.load_config import load_config_file from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts, json_load from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode + logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ class Configuration: :param files: List of file paths :return: configuration dictionary """ - c = Configuration({"config": files}, RunMode.OTHER) + c = Configuration({'config': files}, RunMode.OTHER) return c.get_config() def load_from_files(self, files: List[str]) -> Dict[str, Any]: @@ -123,10 +123,10 @@ class Configuration: the -v/--verbose, --logfile options """ # Log level - config.update({'verbosity': self.args.get("verbosity", 0)}) + config.update({'verbosity': self.args.get('verbosity', 0)}) - if 'logfile' in self.args and self.args["logfile"]: - config.update({'logfile': self.args["logfile"]}) + if 'logfile' in self.args and self.args['logfile']: + config.update({'logfile': self.args['logfile']}) setup_logging(config) @@ -149,22 +149,22 @@ class Configuration: def _process_common_options(self, config: Dict[str, Any]) -> None: # Set strategy if not specified in config and or if it's non default - if self.args.get("strategy") or not config.get('strategy'): - config.update({'strategy': self.args.get("strategy")}) + if self.args.get('strategy') or not config.get('strategy'): + config.update({'strategy': self.args.get('strategy')}) self._args_to_config(config, argname='strategy_path', logstring='Using additional Strategy lookup path: {}') - if ('db_url' in self.args and self.args["db_url"] and - self.args["db_url"] != constants.DEFAULT_DB_PROD_URL): - config.update({'db_url': self.args["db_url"]}) + if ('db_url' in self.args and self.args['db_url'] and + self.args['db_url'] != constants.DEFAULT_DB_PROD_URL): + config.update({'db_url': self.args['db_url']}) logger.info('Parameter --db-url detected ...') if config.get('forcebuy_enable', False): logger.warning('`forcebuy` RPC message enabled.') # Support for sd_notify - if 'sd_notify' in self.args and self.args["sd_notify"]: + if 'sd_notify' in self.args and self.args['sd_notify']: config['internals'].update({'sd_notify': True}) def _process_datadir_options(self, config: Dict[str, Any]) -> None: @@ -173,24 +173,24 @@ class Configuration: --user-data, --datadir """ # Check exchange parameter here - otherwise `datadir` might be wrong. - if "exchange" in self.args and self.args["exchange"]: - config['exchange']['name'] = self.args["exchange"] + if 'exchange' in self.args and self.args['exchange']: + config['exchange']['name'] = self.args['exchange'] logger.info(f"Using exchange {config['exchange']['name']}") if 'pair_whitelist' not in config['exchange']: config['exchange']['pair_whitelist'] = [] - if 'user_data_dir' in self.args and self.args["user_data_dir"]: - config.update({'user_data_dir': self.args["user_data_dir"]}) + if 'user_data_dir' in self.args and self.args['user_data_dir']: + config.update({'user_data_dir': self.args['user_data_dir']}) elif 'user_data_dir' not in config: # Default to cwd/user_data (legacy option ...) - config.update({'user_data_dir': str(Path.cwd() / "user_data")}) + config.update({'user_data_dir': str(Path.cwd() / 'user_data')}) # reset to user_data_dir so this contains the absolute path. config['user_data_dir'] = create_userdata_dir(config['user_data_dir'], create_dir=False) logger.info('Using user-data directory: %s ...', config['user_data_dir']) - config.update({'datadir': create_datadir(config, self.args.get("datadir", None))}) + config.update({'datadir': create_datadir(config, self.args.get('datadir', None))}) logger.info('Using data directory: %s ...', config.get('datadir')) if self.args.get('exportfilename'): @@ -199,18 +199,21 @@ class Configuration: config['exportfilename'] = Path(config['exportfilename']) else: config['exportfilename'] = (config['user_data_dir'] - / 'backtest_results/backtest-result.json') + / 'backtest_results') def _process_optimize_options(self, config: Dict[str, Any]) -> None: # This will override the strategy configuration - self._args_to_config(config, argname='ticker_interval', - logstring='Parameter -i/--ticker-interval detected ... ' - 'Using ticker_interval: {} ...') + self._args_to_config(config, argname='timeframe', + logstring='Parameter -i/--timeframe detected ... ' + 'Using timeframe: {} ...') self._args_to_config(config, argname='position_stacking', logstring='Parameter --enable-position-stacking detected ...') + self._args_to_config( + config, argname='enable_protections', + logstring='Parameter --enable-protections detected, enabling Protections. ...') # Setting max_open_trades to infinite if -1 if config.get('max_open_trades') == -1: config['max_open_trades'] = float('inf') @@ -219,8 +222,8 @@ class Configuration: config.update({'use_max_market_positions': False}) logger.info('Parameter --disable-max-market-positions detected ...') logger.info('max_open_trades set to unlimited ...') - elif 'max_open_trades' in self.args and self.args["max_open_trades"]: - config.update({'max_open_trades': self.args["max_open_trades"]}) + elif 'max_open_trades' in self.args and self.args['max_open_trades']: + config.update({'max_open_trades': self.args['max_open_trades']}) logger.info('Parameter --max-open-trades detected, ' 'overriding max_open_trades to: %s ...', config.get('max_open_trades')) elif config['runmode'] in NON_UTIL_MODES: @@ -242,8 +245,8 @@ class Configuration: self._args_to_config(config, argname='strategy_list', logstring='Using strategy list of {} strategies', logfun=len) - self._args_to_config(config, argname='ticker_interval', - logstring='Overriding ticker interval with Command line argument') + self._args_to_config(config, argname='timeframe', + logstring='Overriding timeframe with Command line argument') self._args_to_config(config, argname='export', logstring='Parameter --export detected: {} ...') @@ -263,6 +266,9 @@ class Configuration: self._args_to_config(config, argname='hyperopt_path', logstring='Using additional Hyperopt lookup path: {}') + self._args_to_config(config, argname='hyperoptexportfilename', + logstring='Using hyperopt file: {}') + self._args_to_config(config, argname='epochs', logstring='Parameter --epochs detected ... ' 'Will run Hyperopt with for {} epochs ...' @@ -295,9 +301,6 @@ class Configuration: self._args_to_config(config, argname='hyperopt_min_trades', logstring='Parameter --min-trades detected: {}') - self._args_to_config(config, argname='hyperopt_continue', - logstring='Hyperopt continue: {}') - self._args_to_config(config, argname='hyperopt_loss', logstring='Using Hyperopt loss class name: {}') @@ -334,6 +337,12 @@ class Configuration: self._args_to_config(config, argname='hyperopt_list_max_total_profit', logstring='Parameter --max-total-profit detected: {}') + self._args_to_config(config, argname='hyperopt_list_min_objective', + logstring='Parameter --min-objective detected: {}') + + self._args_to_config(config, argname='hyperopt_list_max_objective', + logstring='Parameter --max-objective detected: {}') + self._args_to_config(config, argname='hyperopt_list_no_details', logstring='Parameter --no-details detected: {}') @@ -441,12 +450,12 @@ class Configuration: config['pairs'].sort() return - if "config" in self.args and self.args["config"]: + if 'config' in self.args and self.args['config']: logger.info("Using pairlist from configuration.") config['pairs'] = config.get('exchange', {}).get('pair_whitelist') else: # Fall back to /dl_path/pairs.json - pairs_file = config['datadir'] / "pairs.json" + pairs_file = config['datadir'] / 'pairs.json' if pairs_file.exists(): with pairs_file.open('r') as f: config['pairs'] = json_load(f) diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index ff4401c27..6b2a20c8c 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -26,6 +26,24 @@ def check_conflicting_settings(config: Dict[str, Any], ) +def process_removed_setting(config: Dict[str, Any], + section1: str, name1: str, + section2: str, name2: str) -> None: + """ + :param section1: Removed section + :param name1: Removed setting name + :param section2: new section for this key + :param name2: new setting name + """ + section1_config = config.get(section1, {}) + if name1 in section1_config: + raise OperationalException( + f"Setting `{section1}.{name1}` has been moved to `{section2}.{name2}. " + f"Please delete it from your configuration and use the `{section2}.{name2}` " + "setting instead." + ) + + def process_deprecated_setting(config: Dict[str, Any], section1: str, name1: str, section2: str, name2: str) -> None: @@ -44,19 +62,18 @@ def process_deprecated_setting(config: Dict[str, Any], def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: - check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal', - 'experimental', 'use_sell_signal') - check_conflicting_settings(config, 'ask_strategy', 'sell_profit_only', - 'experimental', 'sell_profit_only') - check_conflicting_settings(config, 'ask_strategy', 'ignore_roi_if_buy_signal', - 'experimental', 'ignore_roi_if_buy_signal') + # Kept for future deprecated / moved settings + # check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal', + # 'experimental', 'use_sell_signal') + # process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal', + # 'experimental', 'use_sell_signal') - process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal', - 'experimental', 'use_sell_signal') - process_deprecated_setting(config, 'ask_strategy', 'sell_profit_only', - 'experimental', 'sell_profit_only') - process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal', - 'experimental', 'ignore_roi_if_buy_signal') + process_removed_setting(config, 'experimental', 'use_sell_signal', + 'ask_strategy', 'use_sell_signal') + process_removed_setting(config, 'experimental', 'sell_profit_only', + 'ask_strategy', 'sell_profit_only') + process_removed_setting(config, 'experimental', 'ignore_roi_if_buy_signal', + 'ask_strategy', 'ignore_roi_if_buy_signal') if (config.get('edge', {}).get('enabled', False) and 'capital_available_percentage' in config.get('edge', {})): @@ -67,3 +84,14 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: "'tradable_balance_ratio' and remove 'capital_available_percentage' " "from the edge configuration." ) + if 'ticker_interval' in config: + logger.warning( + "DEPRECATED: " + "Please use 'timeframe' instead of 'ticker_interval." + ) + if 'timeframe' in config: + raise OperationalException( + "Both 'timeframe' and 'ticker_interval' detected." + "Please remove 'ticker_interval' from your configuration to continue operating." + ) + config['timeframe'] = config['ticker_interval'] diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index 6b8c8cb5a..51310f013 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -3,8 +3,9 @@ import shutil from pathlib import Path from typing import Any, Dict, Optional -from freqtrade.exceptions import OperationalException from freqtrade.constants import USER_DATA_FILES +from freqtrade.exceptions import OperationalException + logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index a24ee3d0a..726126034 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -11,6 +11,7 @@ import rapidjson from freqtrade.exceptions import OperationalException + logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index 151003999..32bbd02a0 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -52,11 +52,11 @@ class TimeRange: :return: None (Modifies the object in place) """ if (not self.starttype or (startup_candles - and min_date.timestamp >= self.startts)): + and min_date.int_timestamp >= self.startts)): # If no startts was defined, or backtest-data starts at the defined backtest-date logger.warning("Moving start-date by %s candles to account for startup time.", startup_candles) - self.startts = (min_date.timestamp + timeframe_secs * startup_candles) + self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles) self.starttype = 'date' @staticmethod @@ -89,7 +89,7 @@ class TimeRange: if stype[0]: starts = rvals[index] if stype[0] == 'date' and len(starts) == 8: - start = arrow.get(starts, 'YYYYMMDD').timestamp + start = arrow.get(starts, 'YYYYMMDD').int_timestamp elif len(starts) == 13: start = int(starts) // 1000 else: @@ -98,7 +98,7 @@ class TimeRange: if stype[1]: stops = rvals[index] if stype[1] == 'date' and len(stops) == 8: - stop = arrow.get(stops, 'YYYYMMDD').timestamp + stop = arrow.get(stops, 'YYYYMMDD').int_timestamp elif len(stops) == 13: stop = int(stops) // 1000 else: diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 09ee54208..e7d7e80f6 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -11,7 +11,6 @@ DEFAULT_EXCHANGE = 'bittrex' PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec -DEFAULT_HYPEROPT_LOSS = 'DefaultHyperOptLoss' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' @@ -21,20 +20,31 @@ REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] ORDERBOOK_SIDES = ['ask', 'bid'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] +HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', + 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', + 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', - 'PrecisionFilter', 'PriceFilter', 'ShuffleFilter', 'SpreadFilter'] -AVAILABLE_DATAHANDLERS = ['json', 'jsongz'] + 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', + 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', + 'SpreadFilter'] +AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] +AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 +DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume'] # Don't modify sequence of DEFAULT_TRADES_COLUMNS # it has wide consequences for stored trades files DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost'] +LAST_BT_RESULT_FN = '.last_result.json' + USERPATH_HYPEROPTS = 'hyperopts' USERPATH_STRATEGIES = 'strategies' USERPATH_NOTEBOOKS = 'notebooks' +TELEGRAM_SETTING_OPTIONS = ['on', 'off', 'silent'] + # Soure files with destination directories within user-directory USER_DATA_FILES = { 'sample_strategy.py': USERPATH_STRATEGIES, @@ -71,7 +81,7 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1}, - 'ticker_interval': {'type': 'string'}, + 'timeframe': {'type': 'string'}, 'stake_currency': {'type': 'string'}, 'stake_amount': { 'type': ['number', 'string'], @@ -155,7 +165,9 @@ CONF_SCHEMA = { 'emergencysell': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'stoploss': {'type': 'string', 'enum': ORDERTYPE_POSSIBILITIES}, 'stoploss_on_exchange': {'type': 'boolean'}, - 'stoploss_on_exchange_interval': {'type': 'number'} + 'stoploss_on_exchange_interval': {'type': 'number'}, + 'stoploss_on_exchange_limit_ratio': {'type': 'number', 'minimum': 0.0, + 'maximum': 1.0} }, 'required': ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] }, @@ -172,9 +184,6 @@ CONF_SCHEMA = { 'experimental': { 'type': 'object', 'properties': { - 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'}, - 'ignore_roi_if_buy_signal': {'type': 'boolean'}, 'block_bad_exchanges': {'type': 'boolean'} } }, @@ -184,7 +193,21 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, - 'config': {'type': 'object'} + }, + 'required': ['method'], + } + }, + 'protections': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'method': {'type': 'string', 'enum': AVAILABLE_PROTECTIONS}, + 'stop_duration': {'type': 'number', 'minimum': 0.0}, + 'stop_duration_candles': {'type': 'number', 'minimum': 0}, + 'trade_limit': {'type': 'number', 'minimum': 1}, + 'lookback_period': {'type': 'number', 'minimum': 1}, + 'lookback_period_candles': {'type': 'number', 'minimum': 1}, }, 'required': ['method'], } @@ -195,6 +218,18 @@ CONF_SCHEMA = { 'enabled': {'type': 'boolean'}, 'token': {'type': 'string'}, 'chat_id': {'type': 'string'}, + 'notification_settings': { + 'type': 'object', + 'properties': { + 'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS} + } + } }, 'required': ['enabled', 'token', 'chat_id'] }, @@ -221,6 +256,8 @@ CONF_SCHEMA = { }, 'username': {'type': 'string'}, 'password': {'type': 'string'}, + 'jwt_secret_key': {'type': 'string'}, + 'CORS_origins': {'type': 'array', 'items': {'type': 'string'}}, 'verbosity': {'type': 'string', 'enum': ['error', 'info']}, }, 'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password'] @@ -303,6 +340,7 @@ CONF_SCHEMA = { SCHEMA_TRADE_REQUIRED = [ 'exchange', + 'timeframe', 'max_open_trades', 'stake_currency', 'stake_amount', @@ -329,10 +367,17 @@ SCHEMA_MINIMAL_REQUIRED = [ CANCEL_REASON = { "TIMEOUT": "cancelled due to timeout", - "PARTIALLY_FILLED": "partially filled - keeping order open", + "PARTIALLY_FILLED_KEEP_OPEN": "partially filled - keeping order open", + "PARTIALLY_FILLED": "partially filled", + "FULLY_CANCELLED": "fully cancelled", "ALL_CANCELLED": "cancelled (all unfilled and partially filled open orders cancelled)", "CANCELLED_ON_EXCHANGE": "cancelled on exchange", + "FORCE_SELL": "forcesold", } # List of pairs with their timeframes -ListPairsWithTimeframes = List[Tuple[str, str]] +PairWithTimeframe = Tuple[str, str] +ListPairsWithTimeframes = List[PairWithTimeframe] + +# Type for trades list +TradeList = List[List] diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index d634155b0..ac2f6dc4a 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -2,54 +2,173 @@ Helpers when analyzing backtest data """ import logging +from datetime import datetime, timezone from pathlib import Path -from typing import Dict, Union, Tuple +from typing import Any, Dict, Optional, Tuple, Union import numpy as np import pandas as pd -from datetime import timezone, datetime from scipy.ndimage.interpolation import shift -from freqtrade import persistence +from freqtrade.constants import LAST_BT_RESULT_FN from freqtrade.misc import json_load -from freqtrade.persistence import Trade +from freqtrade.persistence import Trade, init_db + logger = logging.getLogger(__name__) # must align with columns in backtest.py -BT_DATA_COLUMNS = ["pair", "profit_percent", "open_time", "close_time", "index", "duration", +BT_DATA_COLUMNS = ["pair", "profit_percent", "open_date", "close_date", "index", "trade_duration", "open_rate", "close_rate", "open_at_end", "sell_reason"] -def load_backtest_data(filename: Union[Path, str]) -> pd.DataFrame: +def get_latest_optimize_filename(directory: Union[Path, str], variant: str) -> str: """ - Load backtest data file. - :param filename: pathlib.Path object, or string pointing to the file. - :return: a dataframe with the analysis results + Get latest backtest export based on '.last_result.json'. + :param directory: Directory to search for last result + :param variant: 'backtest' or 'hyperopt' - the method to return + :return: string containing the filename of the latest backtest result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content """ - if isinstance(filename, str): - filename = Path(filename) + if isinstance(directory, str): + directory = Path(directory) + if not directory.is_dir(): + raise ValueError(f"Directory '{directory}' does not exist.") + filename = directory / LAST_BT_RESULT_FN if not filename.is_file(): - raise ValueError(f"File {filename} does not exist.") + raise ValueError( + f"Directory '{directory}' does not seem to contain backtest statistics yet.") with filename.open() as file: data = json_load(file) - df = pd.DataFrame(data, columns=BT_DATA_COLUMNS) + if f'latest_{variant}' not in data: + raise ValueError(f"Invalid '{LAST_BT_RESULT_FN}' format.") - df['open_time'] = pd.to_datetime(df['open_time'], - unit='s', - utc=True, - infer_datetime_format=True - ) - df['close_time'] = pd.to_datetime(df['close_time'], - unit='s', - utc=True, - infer_datetime_format=True - ) - df['profit'] = df['close_rate'] - df['open_rate'] - df = df.sort_values("open_time").reset_index(drop=True) + return data[f'latest_{variant}'] + + +def get_latest_backtest_filename(directory: Union[Path, str]) -> str: + """ + Get latest backtest export based on '.last_result.json'. + :param directory: Directory to search for last result + :return: string containing the filename of the latest backtest result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + return get_latest_optimize_filename(directory, 'backtest') + + +def get_latest_hyperopt_filename(directory: Union[Path, str]) -> str: + """ + Get latest hyperopt export based on '.last_result.json'. + :param directory: Directory to search for last result + :return: string containing the filename of the latest hyperopt result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + try: + return get_latest_optimize_filename(directory, 'hyperopt') + except ValueError: + # Return default (legacy) pickle filename + return 'hyperopt_results.pickle' + + +def get_latest_hyperopt_file(directory: Union[Path, str], predef_filename: str = None) -> Path: + """ + Get latest hyperopt export based on '.last_result.json'. + :param directory: Directory to search for last result + :return: string containing the filename of the latest hyperopt result + :raises: ValueError in the following cases: + * Directory does not exist + * `directory/.last_result.json` does not exist + * `directory/.last_result.json` has the wrong content + """ + if isinstance(directory, str): + directory = Path(directory) + if predef_filename: + return directory / predef_filename + return directory / get_latest_hyperopt_filename(directory) + + +def load_backtest_stats(filename: Union[Path, str]) -> Dict[str, Any]: + """ + Load backtest statistics file. + :param filename: pathlib.Path object, or string pointing to the file. + :return: a dictionary containing the resulting file. + """ + if isinstance(filename, str): + filename = Path(filename) + if filename.is_dir(): + filename = filename / get_latest_backtest_filename(filename) + if not filename.is_file(): + raise ValueError(f"File {filename} does not exist.") + logger.info(f"Loading backtest result from {filename}") + with filename.open() as file: + data = json_load(file) + + return data + + +def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = None) -> pd.DataFrame: + """ + Load backtest data file. + :param filename: pathlib.Path object, or string pointing to a file or directory + :param strategy: Strategy to load - mainly relevant for multi-strategy backtests + Can also serve as protection to load the correct result. + :return: a dataframe with the analysis results + :raise: ValueError if loading goes wrong. + """ + data = load_backtest_stats(filename) + if not isinstance(data, list): + # new, nested format + if 'strategy' not in data: + raise ValueError("Unknown dataformat.") + + if not strategy: + if len(data['strategy']) == 1: + strategy = list(data['strategy'].keys())[0] + else: + raise ValueError("Detected backtest result with more than one strategy. " + "Please specify a strategy.") + + if strategy not in data['strategy']: + raise ValueError(f"Strategy {strategy} not available in the backtest result.") + + data = data['strategy'][strategy]['trades'] + df = pd.DataFrame(data) + df['open_date'] = pd.to_datetime(df['open_date'], + utc=True, + infer_datetime_format=True + ) + df['close_date'] = pd.to_datetime(df['close_date'], + utc=True, + infer_datetime_format=True + ) + else: + # old format - only with lists. + df = pd.DataFrame(data, columns=BT_DATA_COLUMNS) + + df['open_date'] = pd.to_datetime(df['open_date'], + unit='s', + utc=True, + infer_datetime_format=True + ) + df['close_date'] = pd.to_datetime(df['close_date'], + unit='s', + utc=True, + infer_datetime_format=True + ) + df['profit_abs'] = df['close_rate'] - df['open_rate'] + df = df.sort_values("open_date").reset_index(drop=True) return df @@ -64,12 +183,11 @@ def analyze_trade_parallelism(results: pd.DataFrame, timeframe: str) -> pd.DataF from freqtrade.exchange import timeframe_to_minutes timeframe_min = timeframe_to_minutes(timeframe) # compute how long each trade was left outstanding as date indexes - dates = [pd.Series(pd.date_range(row[1].open_time, row[1].close_time, + dates = [pd.Series(pd.date_range(row[1]['open_date'], row[1]['close_date'], freq=f"{timeframe_min}min")) - for row in results[['open_time', 'close_time']].iterrows()] - # track the lifetime of each trade in number of candles + for row in results[['open_date', 'close_date']].iterrows()] deltas = [len(x) for x in dates] - # concat expands and flattens the list of lists of dates + # expand and flatten the list of lists of dates dates = pd.Series(pd.concat(dates).values, name='date') # trades are repeated (column wise) according to their lifetime df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0), columns=results.columns) @@ -98,20 +216,25 @@ def evaluate_result_multi(results: pd.DataFrame, timeframe: str, return df_final[df_final['open_trades'] > max_open_trades] -def load_trades_from_db(db_url: str) -> pd.DataFrame: +def load_trades_from_db(db_url: str, strategy: Optional[str] = None) -> pd.DataFrame: """ Load trades from a DB (using dburl) :param db_url: Sqlite url (default format sqlite:///tradesv3.dry-run.sqlite) + :param strategy: Strategy to load - mainly relevant for multi-strategy backtests + Can also serve as protection to load the correct result. :return: Dataframe containing Trades """ - trades: pd.DataFrame = pd.DataFrame([], columns=BT_DATA_COLUMNS) - persistence.init(db_url, clean_open_orders=False) + init_db(db_url, clean_open_orders=False) - columns = ["pair", "open_time", "close_time", "profit", "profit_percent", - "open_rate", "close_rate", "amount", "duration", "sell_reason", + columns = ["pair", "open_date", "close_date", "profit", "profit_percent", + "open_rate", "close_rate", "amount", "trade_duration", "sell_reason", "fee_open", "fee_close", "open_rate_requested", "close_rate_requested", "stake_amount", "max_rate", "min_rate", "id", "exchange", - "stop_loss", "initial_stop_loss", "strategy", "ticker_interval"] + "stop_loss", "initial_stop_loss", "strategy", "timeframe"] + + filters = [] + if strategy: + filters.append(Trade.strategy == strategy) trades = pd.DataFrame([(t.pair, t.open_date.replace(tzinfo=timezone.utc), @@ -129,18 +252,18 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: t.min_rate, t.id, t.exchange, t.stop_loss, t.initial_stop_loss, - t.strategy, t.ticker_interval + t.strategy, t.timeframe ) - for t in Trade.get_trades().all()], + for t in Trade.get_trades(filters).all()], columns=columns) return trades def load_trades(source: str, db_url: str, exportfilename: Path, - no_trades: bool = False) -> pd.DataFrame: + no_trades: bool = False, strategy: Optional[str] = None) -> pd.DataFrame: """ - Based on configuration option "trade_source": + Based on configuration option 'trade_source': * loads data from DB (using `db_url`) * loads data from backtestfile (using `exportfilename`) :param source: "DB" or "file" - specify source to load from @@ -156,7 +279,7 @@ def load_trades(source: str, db_url: str, exportfilename: Path, if source == "DB": return load_trades_from_db(db_url) elif source == "file": - return load_backtest_data(exportfilename) + return load_backtest_data(exportfilename, strategy) def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame, @@ -171,11 +294,31 @@ def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame, else: trades_start = dataframe.iloc[0]['date'] trades_stop = dataframe.iloc[-1]['date'] - trades = trades.loc[(trades['open_time'] >= trades_start) & - (trades['close_time'] <= trades_stop)] + trades = trades.loc[(trades['open_date'] >= trades_start) & + (trades['close_date'] <= trades_stop)] return trades +def calculate_market_change(data: Dict[str, pd.DataFrame], column: str = "close") -> float: + """ + Calculate market change based on "column". + Calculation is done by taking the first non-null and the last non-null element of each column + and calculating the pctchange as "(last - first) / first". + Then the results per pair are combined as mean. + + :param data: Dict of Dataframes, dict key should be pair. + :param column: Column in the original dataframes to use + :return: + """ + tmp_means = [] + for pair, df in data.items(): + start = df[column].dropna().iloc[0] + end = df[column].dropna().iloc[-1] + tmp_means.append((end - start) / start) + + return np.mean(tmp_means) + + def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], column: str = "close") -> pd.DataFrame: """ @@ -198,7 +341,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, """ Adds a column `col_name` with the cumulative profit for the given trades array. :param df: DataFrame with date index - :param trades: DataFrame containing trades (requires columns close_time and profit_percent) + :param trades: DataFrame containing trades (requires columns close_date and profit_percent) :param col_name: Column name that will be assigned the results :param timeframe: Timeframe used during the operations :return: Returns df with one additional column, col_name, containing the cumulative profit. @@ -209,7 +352,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, from freqtrade.exchange import timeframe_to_minutes timeframe_minutes = timeframe_to_minutes(timeframe) # Resample to timeframe to make sure trades match candles - _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_time' + _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date' )[['profit_percent']].sum() df.loc[:, col_name] = _trades_sum.cumsum() # Set first value to 0 @@ -219,13 +362,13 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, return df -def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time', +def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_date', value_col: str = 'profit_percent' ) -> Tuple[float, pd.Timestamp, pd.Timestamp]: """ Calculate max drawdown and the corresponding close dates - :param trades: DataFrame containing trades (requires columns close_time and profit_percent) - :param date_col: Column in DataFrame to use for dates (defaults to 'close_time') + :param trades: DataFrame containing trades (requires columns close_date and profit_percent) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_date') :param value_col: Column in DataFrame to use for values (defaults to 'profit_percent') :return: Tuple (float, highdate, lowdate) with absolute max drawdown, high and low time :raise: ValueError if trade-dataframe was found empty. diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index f73775432..d4053abaa 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -10,8 +10,8 @@ from typing import Any, Dict, List import pandas as pd from pandas import DataFrame, to_datetime -from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, - DEFAULT_TRADES_COLUMNS) +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList + logger = logging.getLogger(__name__) @@ -168,7 +168,7 @@ def trades_remove_duplicates(trades: List[List]) -> List[List]: return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))] -def trades_dict_to_list(trades: List[Dict]) -> List[List]: +def trades_dict_to_list(trades: List[Dict]) -> TradeList: """ Convert fetch_trades result into a List (to be more memory efficient). :param trades: List of trades, as returned by ccxt.fetch_trades. @@ -177,16 +177,18 @@ def trades_dict_to_list(trades: List[Dict]) -> List[List]: return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades] -def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame: +def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame: """ Converts trades list to OHLCV list - TODO: This should get a dedicated test :param trades: List of trades, as returned by ccxt.fetch_trades. :param timeframe: Timeframe to resample data to :return: OHLCV Dataframe. + :raises: ValueError if no trades are provided """ from freqtrade.exchange import timeframe_to_minutes timeframe_minutes = timeframe_to_minutes(timeframe) + if not trades: + raise ValueError('Trade-list empty.') df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS) df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True,) @@ -236,12 +238,12 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: from freqtrade.data.history.idatahandler import get_datahandler src = get_datahandler(config['datadir'], convert_from) trg = get_datahandler(config['datadir'], convert_to) - timeframes = config.get('timeframes', [config.get('ticker_interval')]) + timeframes = config.get('timeframes', [config.get('timeframe')]) logger.info(f"Converting candle (OHLCV) for timeframe {timeframes}") if 'pairs' not in config: config['pairs'] = [] - # Check timeframes or fall back to ticker_interval. + # Check timeframes or fall back to timeframe. for timeframe in timeframes: config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'], timeframe)) @@ -255,7 +257,8 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: drop_incomplete=False, startup_candles=0) logger.info(f"Converting {len(data)} candles for {pair}") - trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) - if erase and convert_from != convert_to: - logger.info(f"Deleting source data for {pair} / {timeframe}") - src.ohlcv_purge(pair=pair, timeframe=timeframe) + if len(data) > 0: + trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source data for {pair} / {timeframe}") + src.ohlcv_purge(pair=pair, timeframe=timeframe) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index a01344364..a035b7c3b 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -5,15 +5,16 @@ including ticker and orderbook data, live and historical candle (OHLCV) data Common Interface for bot and strategy to access data. """ import logging -from typing import Any, Dict, List, Optional +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame +from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe from freqtrade.data.history import load_pair_history -from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.exchange import Exchange from freqtrade.state import RunMode -from freqtrade.constants import ListPairsWithTimeframes logger = logging.getLogger(__name__) @@ -25,6 +26,24 @@ class DataProvider: self._config = config self._exchange = exchange self._pairlists = pairlists + self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {} + + def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None: + """ + Store cached Dataframe. + Using private method as this should never be used by a user + (but the class is exposed via `self.dp` to the strategy) + :param pair: pair to get the data for + :param timeframe: Timeframe to get data for + :param dataframe: analyzed dataframe + """ + self.__cached_pairs[(pair, timeframe)] = (dataframe, datetime.now(timezone.utc)) + + def add_pairlisthandler(self, pairlists) -> None: + """ + Allow adding pairlisthandler after initialization + """ + self._pairlists = pairlists def refresh(self, pairlist: ListPairsWithTimeframes, @@ -55,7 +74,7 @@ class DataProvider: Use False only for read-only operations (where the dataframe is not modified) """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - return self._exchange.klines((pair, timeframe or self._config['ticker_interval']), + return self._exchange.klines((pair, timeframe or self._config['timeframe']), copy=copy) else: return DataFrame() @@ -67,8 +86,9 @@ class DataProvider: :param timeframe: timeframe to get data for """ return load_pair_history(pair=pair, - timeframe=timeframe or self._config['ticker_interval'], - datadir=self._config['datadir'] + timeframe=timeframe or self._config['timeframe'], + datadir=self._config['datadir'], + data_format=self._config.get('dataformat_ohlcv', 'json') ) def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame: @@ -89,6 +109,20 @@ class DataProvider: logger.warning(f"No data found for ({pair}, {timeframe}).") return data + def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]: + """ + :param pair: pair to get the data for + :param timeframe: timeframe to get data for + :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe + combination. + Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. + """ + if (pair, timeframe) in self.__cached_pairs: + return self.__cached_pairs[(pair, timeframe)] + else: + + return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc)) + def market(self, pair: str) -> Optional[Dict[str, Any]]: """ Return market data for the pair @@ -105,7 +139,7 @@ class DataProvider: """ try: return self._exchange.fetch_ticker(pair) - except DependencyException: + except ExchangeError: return {} def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: diff --git a/freqtrade/data/history/__init__.py b/freqtrade/data/history/__init__.py index 23f635a98..107f9c401 100644 --- a/freqtrade/data/history/__init__.py +++ b/freqtrade/data/history/__init__.py @@ -5,10 +5,8 @@ Includes: * load data for a pair (or a list of pairs) from disk * download data from exchange and store to disk """ - -from .history_utils import (convert_trades_to_ohlcv, # noqa: F401 - get_timerange, load_data, load_pair_history, - refresh_backtest_ohlcv_data, - refresh_backtest_trades_data, refresh_data, +# flake8: noqa: F401 +from .history_utils import (convert_trades_to_ohlcv, get_timerange, load_data, load_pair_history, + refresh_backtest_ohlcv_data, refresh_backtest_trades_data, refresh_data, validate_backtest_data) -from .idatahandler import get_datahandler # noqa: F401 +from .idatahandler import get_datahandler diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py new file mode 100644 index 000000000..d116637e7 --- /dev/null +++ b/freqtrade/data/history/hdf5datahandler.py @@ -0,0 +1,213 @@ +import logging +import re +from pathlib import Path +from typing import List, Optional + +import numpy as np +import pandas as pd + +from freqtrade import misc +from freqtrade.configuration import TimeRange +from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, + ListPairsWithTimeframes, TradeList) + +from .idatahandler import IDataHandler + + +logger = logging.getLogger(__name__) + + +class HDF5DataHandler(IDataHandler): + + _columns = DEFAULT_DATAFRAME_COLUMNS + + @classmethod + def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :return: List of Tuples of (pair, timeframe) + """ + _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.h5)', p.name) + for p in datadir.glob("*.h5")] + return [(match[1].replace('_', '/'), match[2]) for match in _tmp + if match and len(match.groups()) > 1] + + @classmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs with ohlcv data available in this datadir + for the specified timeframe + :param datadir: Directory to search for ohlcv files + :param timeframe: Timeframe to search pairs for + :return: List of Pairs + """ + + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.h5)', p.name) + for p in datadir.glob(f"*{timeframe}.h5")] + # Check if regex found something and only return these results + return [match[0].replace('_', '/') for match in _tmp if match] + + def ohlcv_store(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: + """ + Store data in hdf5 file. + :param pair: Pair - used to generate filename + :timeframe: Timeframe - used to generate filename + :data: Dataframe containing OHLCV data + :return: None + """ + key = self._pair_ohlcv_key(pair, timeframe) + _data = data.copy() + + filename = self._pair_data_filename(self._datadir, pair, timeframe) + + ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc') + ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date']) + + ds.close() + + def _ohlcv_load(self, pair: str, timeframe: str, + timerange: Optional[TimeRange] = None) -> pd.DataFrame: + """ + Internal method used to load data for one pair from disk. + Implements the loading and conversion to a Pandas dataframe. + Timerange trimming and dataframe validation happens outside of this method. + :param pair: Pair to load data + :param timeframe: Timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. + :return: DataFrame with ohlcv data, or empty DataFrame + """ + key = self._pair_ohlcv_key(pair, timeframe) + filename = self._pair_data_filename(self._datadir, pair, timeframe) + + if not filename.exists(): + return pd.DataFrame(columns=self._columns) + where = [] + if timerange: + if timerange.starttype == 'date': + where.append(f"date >= Timestamp({timerange.startts * 1e9})") + if timerange.stoptype == 'date': + where.append(f"date < Timestamp({timerange.stopts * 1e9})") + + pairdata = pd.read_hdf(filename, key=key, mode="r", where=where) + + if list(pairdata.columns) != self._columns: + raise ValueError("Wrong dataframe format") + pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) + return pairdata + + def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :param timeframe: Timeframe (e.g. "5m") + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if filename.exists(): + filename.unlink() + return True + return False + + def ohlcv_append(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: + """ + Append data to existing data structures + :param pair: Pair + :param timeframe: Timeframe this ohlcv data is for + :param data: Data to append. + """ + raise NotImplementedError() + + @classmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs for which trade data is available in this + :param datadir: Directory to search for ohlcv files + :return: List of Pairs + """ + _tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name) + for p in datadir.glob("*trades.h5")] + # Check if regex found something and only return these results to avoid exceptions. + return [match[0].replace('_', '/') for match in _tmp if match] + + def trades_store(self, pair: str, data: TradeList) -> None: + """ + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + key = self._pair_trades_key(pair) + + ds = pd.HDFStore(self._pair_trades_filename(self._datadir, pair), + mode='a', complevel=9, complib='blosc') + ds.put(key, pd.DataFrame(data, columns=DEFAULT_TRADES_COLUMNS), + format='table', data_columns=['timestamp']) + ds.close() + + def trades_append(self, pair: str, data: TradeList): + """ + Append data to existing files + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + raise NotImplementedError() + + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from h5 file. + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + key = self._pair_trades_key(pair) + filename = self._pair_trades_filename(self._datadir, pair) + + if not filename.exists(): + return [] + where = [] + if timerange: + if timerange.starttype == 'date': + where.append(f"timestamp >= {timerange.startts * 1e3}") + if timerange.stoptype == 'date': + where.append(f"timestamp < {timerange.stopts * 1e3}") + + trades: pd.DataFrame = pd.read_hdf(filename, key=key, mode="r", where=where) + trades[['id', 'type']] = trades[['id', 'type']].replace({np.nan: None}) + return trades.values.tolist() + + def trades_purge(self, pair: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_trades_filename(self._datadir, pair) + if filename.exists(): + filename.unlink() + return True + return False + + @classmethod + def _pair_ohlcv_key(cls, pair: str, timeframe: str) -> str: + return f"{pair}/ohlcv/tf_{timeframe}" + + @classmethod + def _pair_trades_key(cls, pair: str) -> str: + return f"{pair}/trades" + + @classmethod + def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-{timeframe}.h5') + return filename + + @classmethod + def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-trades.h5') + return filename diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 4f3f75a87..3b8b5a2f0 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -9,14 +9,14 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS -from freqtrade.data.converter import (ohlcv_to_dataframe, - trades_remove_duplicates, - trades_to_ohlcv) +from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, + trades_remove_duplicates, trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.misc import format_ms_time + logger = logging.getLogger(__name__) @@ -135,7 +135,6 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona start = None if timerange: if timerange.starttype == 'date': - # TODO: convert to date for conversion start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) # Intentionally don't pass timerange in - since we need to load the full dataset. @@ -202,7 +201,10 @@ def _download_pair_history(datadir: Path, if data.empty: data = new_dataframe else: - data = data.append(new_dataframe) + # Run cleaning again to ensure there were no duplicate candles + # Especially between existing and new data. + data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair, + fill_missing=False, drop_incomplete=False) logger.debug("New Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') @@ -212,10 +214,9 @@ def _download_pair_history(datadir: Path, data_handler.ohlcv_store(pair, timeframe, data=data) return True - except Exception as e: - logger.error( - f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}. ' - f'Error: {e}' + except Exception: + logger.exception( + f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}.' ) return False @@ -270,6 +271,11 @@ def _download_trades_history(exchange: Exchange, # DEFAULT_TRADES_COLUMNS: 0 -> timestamp # DEFAULT_TRADES_COLUMNS: 1 -> id + if trades and since < trades[0][0]: + # since is before the first trade + logger.info(f"Start earlier than available data. Redownloading trades for {pair}...") + trades = [] + from_id = trades[-1][1] if trades else None if trades and since < trades[-1][0]: # Reset since to the last available point @@ -297,10 +303,9 @@ def _download_trades_history(exchange: Exchange, logger.info(f"New Amount of trades: {len(trades)}") return True - except Exception as e: - logger.error( + except Exception: + logger.exception( f'Failed to download historic trades for pair: "{pair}". ' - f'Error: {e}' ) return False @@ -349,9 +354,12 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], if erase: if data_handler_ohlcv.ohlcv_purge(pair, timeframe): logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') - ohlcv = trades_to_ohlcv(trades, timeframe) - # Store ohlcv - data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) + try: + ohlcv = trades_to_ohlcv(trades, timeframe) + # Store ohlcv + data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) + except ValueError: + logger.exception(f'Could not convert {pair} to OHLCV.') def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index d5d7c16db..070d9039d 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -13,14 +13,12 @@ from typing import List, Optional, Type from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.data.converter import (clean_ohlcv_dataframe, - trades_remove_duplicates, trim_dataframe) +from freqtrade.constants import ListPairsWithTimeframes, TradeList +from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe from freqtrade.exchange import timeframe_to_seconds -logger = logging.getLogger(__name__) -# Type for trades list -TradeList = List[List] +logger = logging.getLogger(__name__) class IDataHandler(ABC): @@ -28,6 +26,14 @@ class IDataHandler(ABC): def __init__(self, datadir: Path) -> None: self._datadir = datadir + @abstractclassmethod + def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :return: List of Tuples of (pair, timeframe) + """ + @abstractclassmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: """ @@ -41,9 +47,7 @@ class IDataHandler(ABC): @abstractmethod def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: """ - Store data in json format "values". - format looks as follows: - [[,,,,]] + Store ohlcv data. :param pair: Pair - used to generate filename :timeframe: Timeframe - used to generate filename :data: Dataframe containing OHLCV data @@ -230,6 +234,9 @@ def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: elif datatype == 'jsongz': from .jsondatahandler import JsonGzDataHandler return JsonGzDataHandler + elif datatype == 'hdf5': + from .hdf5datahandler import HDF5DataHandler + return HDF5DataHandler else: raise ValueError(f"No datahandler for datatype {datatype} available.") diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 01320f129..9122170d5 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -8,10 +8,11 @@ from pandas import DataFrame, read_json, to_datetime from freqtrade import misc from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes, TradeList from freqtrade.data.converter import trades_dict_to_list -from .idatahandler import IDataHandler, TradeList +from .idatahandler import IDataHandler + logger = logging.getLogger(__name__) @@ -21,6 +22,18 @@ class JsonDataHandler(IDataHandler): _use_zip = False _columns = DEFAULT_DATAFRAME_COLUMNS + @classmethod + def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + """ + Returns a list of all pairs with ohlcv data available in this datadir + :param datadir: Directory to search for ohlcv files + :return: List of Tuples of (pair, timeframe) + """ + _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name) + for p in datadir.glob(f"*.{cls._get_file_extension()}")] + return [(match[1].replace('_', '/'), match[2]) for match in _tmp + if match and len(match.groups()) > 1] + @classmethod def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: """ diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 2550cf604..037717c68 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -9,11 +9,12 @@ import utils_find_1st as utf1st from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.constants import UNLIMITED_STAKE_AMOUNT -from freqtrade.exceptions import OperationalException +from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT from freqtrade.data.history import get_timerange, load_data, refresh_data +from freqtrade.exceptions import OperationalException from freqtrade.strategy.interface import SellType + logger = logging.getLogger(__name__) @@ -86,7 +87,7 @@ class Edge: heartbeat = self.edge_config.get('process_throttle_secs') if (self._last_updated > 0) and ( - self._last_updated + heartbeat > arrow.utcnow().timestamp): + self._last_updated + heartbeat > arrow.utcnow().int_timestamp): return False data: Dict[str, Any] = {} @@ -98,14 +99,14 @@ class Edge: datadir=self.config['datadir'], pairs=pairs, exchange=self.exchange, - timeframe=self.strategy.ticker_interval, + timeframe=self.strategy.timeframe, timerange=self._timerange, ) data = load_data( datadir=self.config['datadir'], pairs=pairs, - timeframe=self.strategy.ticker_interval, + timeframe=self.strategy.timeframe, timerange=self._timerange, startup_candles=self.strategy.startup_candle_count, data_format=self.config.get('dataformat_ohlcv', 'json'), @@ -121,12 +122,9 @@ class Edge: # Print timeframe min_date, max_date = get_timerange(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days) ...', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) + logger.info(f'Measuring data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'({(max_date - min_date).days} days)..') headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] trades: list = [] @@ -148,7 +146,7 @@ class Edge: # Fill missing, calculable columns, profit, duration , abs etc. trades_df = self._fill_calculable_fields(DataFrame(trades)) self._cached_pairs = self._process_expectancy(trades_df) - self._last_updated = arrow.utcnow().timestamp + self._last_updated = arrow.utcnow().int_timestamp return True @@ -240,7 +238,7 @@ class Edge: # All returned values are relative, they are defined as ratios. stake = 0.015 - result['trade_duration'] = result['close_time'] - result['open_time'] + result['trade_duration'] = result['close_date'] - result['open_date'] result['trade_duration'] = result['trade_duration'].map( lambda x: int(x.total_seconds() / 60)) @@ -281,8 +279,8 @@ class Edge: # # Removing Pumps if self.edge_config.get('remove_pumps', False): - results = results.groupby(['pair', 'stoploss']).apply( - lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) + results = results[results['profit_abs'] < 2 * results['profit_abs'].std() + + results['profit_abs'].mean()] ########################################################################## # Removing trades having a duration more than X minutes (set in config) @@ -312,8 +310,10 @@ class Edge: # Calculating number of losing trades, average win and average loss df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] - df['average_win'] = df['profit_sum'] / df['nb_win_trades'] - df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] + df['average_win'] = np.where(df['nb_win_trades'] == 0, 0.0, + df['profit_sum'] / df['nb_win_trades']) + df['average_loss'] = np.where(df['nb_loss_trades'] == 0, 0.0, + df['loss_sum'] / df['nb_loss_trades']) # Win rate = number of profitable trades / number of trades df['winrate'] = df['nb_win_trades'] / df['nb_trades'] @@ -430,10 +430,8 @@ class Edge: 'stoploss': stoploss, 'profit_ratio': '', 'profit_abs': '', - 'open_time': date_column[open_trade_index], - 'close_time': date_column[exit_index], - 'open_index': start_point + open_trade_index, - 'close_index': start_point + exit_index, + 'open_date': date_column[open_trade_index], + 'close_date': date_column[exit_index], 'trade_duration': '', 'open_rate': round(open_price, 15), 'close_rate': round(exit_price, 15), diff --git a/freqtrade/exceptions.py b/freqtrade/exceptions.py index 7cfed87e8..caf970606 100644 --- a/freqtrade/exceptions.py +++ b/freqtrade/exceptions.py @@ -29,7 +29,14 @@ class PricingError(DependencyException): """ -class InvalidOrderException(FreqtradeException): +class ExchangeError(DependencyException): + """ + Error raised out of the exchange. + Has multiple Errors to determine the appropriate error. + """ + + +class InvalidOrderException(ExchangeError): """ This is returned when the order is not valid. Example: If stoploss on exchange order is hit, then trying to cancel the order @@ -37,7 +44,21 @@ class InvalidOrderException(FreqtradeException): """ -class TemporaryError(FreqtradeException): +class RetryableOrderError(InvalidOrderException): + """ + This is returned when the order is not found. + This Error will be repeated with increasing backof (in line with DDosError). + """ + + +class InsufficientFundsError(InvalidOrderException): + """ + This error is used when there are not enough funds available on the exchange + to create an order. + """ + + +class TemporaryError(ExchangeError): """ Temporary network or exchange related error. This could happen when an exchange is congested, unavailable, or the user @@ -45,6 +66,13 @@ class TemporaryError(FreqtradeException): """ +class DDosProtection(TemporaryError): + """ + Temporary error caused by DDOS protection. + Bot will wait for a second and then retry. + """ + + class StrategyError(FreqtradeException): """ Errors with custom user-code deteced. diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index a39f8f5df..15ba7b9f6 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,20 +1,17 @@ # flake8: noqa: F401 +# isort: off from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS from freqtrade.exchange.exchange import Exchange -from freqtrade.exchange.exchange import (get_exchange_bad_reason, - is_exchange_bad, - is_exchange_known_ccxt, - is_exchange_officially_supported, - ccxt_exchanges, - available_exchanges) -from freqtrade.exchange.exchange import (timeframe_to_seconds, - timeframe_to_minutes, - timeframe_to_msecs, - timeframe_to_next_date, - timeframe_to_prev_date) -from freqtrade.exchange.exchange import (market_is_active, - symbol_is_pair) -from freqtrade.exchange.kraken import Kraken -from freqtrade.exchange.binance import Binance +# isort: on from freqtrade.exchange.bibox import Bibox +from freqtrade.exchange.binance import Binance +from freqtrade.exchange.bittrex import Bittrex +from freqtrade.exchange.bybit import Bybit +from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges, + get_exchange_bad_reason, is_exchange_bad, + is_exchange_known_ccxt, is_exchange_officially_supported, + market_is_active, timeframe_to_minutes, timeframe_to_msecs, + timeframe_to_next_date, timeframe_to_prev_date, + timeframe_to_seconds) from freqtrade.exchange.ftx import Ftx +from freqtrade.exchange.kraken import Kraken diff --git a/freqtrade/exchange/bibox.py b/freqtrade/exchange/bibox.py index 229abe766..f0c2dd00b 100644 --- a/freqtrade/exchange/bibox.py +++ b/freqtrade/exchange/bibox.py @@ -4,6 +4,7 @@ from typing import Dict from freqtrade.exchange import Exchange + logger = logging.getLogger(__name__) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 4279f392c..26ec30a8a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -4,9 +4,11 @@ from typing import Dict import ccxt -from freqtrade.exceptions import (DependencyException, InvalidOrderException, +from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Exchange +from freqtrade.exchange.common import retrier + logger = logging.getLogger(__name__) @@ -16,22 +18,12 @@ class Binance(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, "order_time_in_force": ['gtc', 'fok', 'ioc'], + "ohlcv_candle_limit": 1000, "trades_pagination": "id", "trades_pagination_arg": "fromId", + "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000], } - def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: - """ - get order book level 2 from exchange - - 20180619: binance support limits but only on specific range - """ - limit_range = [5, 10, 20, 50, 100, 500, 1000] - # get next-higher step in the limit_range list - limit = min(list(filter(lambda x: limit <= x, limit_range))) - - return super().fetch_l2_order_book(pair, limit) - def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: """ Verify stop_loss against stoploss-order value (limit or price) @@ -39,6 +31,7 @@ class Binance(Exchange): """ return order['type'] == 'stop_loss_limit' and stop_loss > float(order['info']['stopPrice']) + @retrier(retries=0) def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: """ creates a stoploss limit order. @@ -77,8 +70,8 @@ class Binance(Exchange): 'stop price: %s. limit: %s', pair, stop_price, rate) return order except ccxt.InsufficientFunds as e: - raise DependencyException( - f'Insufficient funds to create {ordertype} sell order on market {pair}.' + raise InsufficientFundsError( + f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Tried to sell amount {amount} at rate {rate}. ' f'Message: {e}') from e except ccxt.InvalidOrder as e: @@ -88,6 +81,8 @@ class Binance(Exchange): f'Could not create {ordertype} sell order on market {pair}. ' f'Tried to sell amount {amount} at rate {rate}. ' f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e diff --git a/freqtrade/exchange/bittrex.py b/freqtrade/exchange/bittrex.py new file mode 100644 index 000000000..4318f9cf0 --- /dev/null +++ b/freqtrade/exchange/bittrex.py @@ -0,0 +1,23 @@ +""" Bittrex exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Bittrex(Exchange): + """ + Bittrex exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + _ft_has: Dict = { + "l2_limit_range": [1, 25, 500], + } diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py new file mode 100644 index 000000000..4a44bb42d --- /dev/null +++ b/freqtrade/exchange/bybit.py @@ -0,0 +1,24 @@ +""" Bybit exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Bybit(Exchange): + """ + Bybit exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + # fetchCurrencies API point requires authentication for Bybit, + _ft_has: Dict = { + "ohlcv_candle_limit": 200, + } diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index a10d41247..ce0fde9e4 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -1,17 +1,26 @@ +import asyncio import logging +import time +from functools import wraps + +from freqtrade.exceptions import DDosProtection, RetryableOrderError, TemporaryError -from freqtrade.exceptions import TemporaryError logger = logging.getLogger(__name__) +# Maximum default retry count. +# Functions are always called RETRY_COUNT + 1 times (for the original call) API_RETRY_COUNT = 4 +API_FETCH_ORDER_RETRY_COUNT = 5 + BAD_EXCHANGES = { "bitmex": "Various reasons.", "bitstamp": "Does not provide history. " "Details in https://github.com/freqtrade/freqtrade/issues/1983", "hitbtc": "This API cannot be used with Freqtrade. " "Use `hitbtc2` exchange id to access this exchange.", + "phemex": "Does not provide history. ", **dict.fromkeys([ 'adara', 'anxpro', @@ -88,6 +97,13 @@ MAP_EXCHANGE_CHILDCLASS = { } +def calculate_backoff(retrycount, max_retries): + """ + Calculate backoff + """ + return (max_retries - retrycount) ** 2 + 1 + + def retrier_async(f): async def wrapper(*args, **kwargs): count = kwargs.pop('count', API_RETRY_COUNT) @@ -96,9 +112,13 @@ def retrier_async(f): except TemporaryError as ex: logger.warning('%s() returned exception: "%s"', f.__name__, ex) if count > 0: + logger.warning('retrying %s() still for %s times', f.__name__, count) count -= 1 kwargs.update({'count': count}) - logger.warning('retrying %s() still for %s times', f.__name__, count) + if isinstance(ex, DDosProtection): + backoff_delay = calculate_backoff(count + 1, API_RETRY_COUNT) + logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}") + await asyncio.sleep(backoff_delay) return await wrapper(*args, **kwargs) else: logger.warning('Giving up retrying: %s()', f.__name__) @@ -106,19 +126,31 @@ def retrier_async(f): return wrapper -def retrier(f): - def wrapper(*args, **kwargs): - count = kwargs.pop('count', API_RETRY_COUNT) - try: - return f(*args, **kwargs) - except TemporaryError as ex: - logger.warning('%s() returned exception: "%s"', f.__name__, ex) - if count > 0: - count -= 1 - kwargs.update({'count': count}) - logger.warning('retrying %s() still for %s times', f.__name__, count) - return wrapper(*args, **kwargs) - else: - logger.warning('Giving up retrying: %s()', f.__name__) - raise ex - return wrapper +def retrier(_func=None, retries=API_RETRY_COUNT): + def decorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + count = kwargs.pop('count', retries) + try: + return f(*args, **kwargs) + except (TemporaryError, RetryableOrderError) as ex: + logger.warning('%s() returned exception: "%s"', f.__name__, ex) + if count > 0: + logger.warning('retrying %s() still for %s times', f.__name__, count) + count -= 1 + kwargs.update({'count': count}) + if isinstance(ex, DDosProtection) or isinstance(ex, RetryableOrderError): + # increasing backoff + backoff_delay = calculate_backoff(count + 1, retries) + logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}") + time.sleep(backoff_delay) + return wrapper(*args, **kwargs) + else: + logger.warning('Giving up retrying: %s()', f.__name__) + raise ex + return wrapper + # Support both @retrier and @retrier(retries=2) syntax + if _func is None: + return decorator + else: + return decorator(_func) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 038fc22bc..6f495e605 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -8,22 +8,24 @@ import logging from copy import deepcopy from datetime import datetime, timezone from math import ceil -from random import randint from typing import Any, Dict, List, Optional, Tuple import arrow import ccxt import ccxt.async_support as ccxt_async -from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, - TRUNCATE, decimal_to_precision) +from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, + decimal_to_precision) from pandas import DataFrame -from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list -from freqtrade.exceptions import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) -from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async -from freqtrade.misc import deep_merge_dicts, safe_value_fallback from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list +from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, + InvalidOrderException, OperationalException, RetryableOrderError, + TemporaryError) +from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier, + retrier_async) +from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 + CcxtModuleType = Any @@ -51,7 +53,7 @@ class Exchange: "ohlcv_partial_candle": True, "trades_pagination": "time", # Possible are "time" or "id" "trades_pagination_arg": "since", - + "l2_limit_range": None, } _ft_has: Dict = {} @@ -79,13 +81,13 @@ class Exchange: if config['dry_run']: logger.info('Instance is running with dry_run enabled') - + logger.info(f"Using CCXT {ccxt.__version__}") exchange_config = config['exchange'] # Deep merge ft_has with default ft_has options self._ft_has = deep_merge_dicts(self._ft_has, deepcopy(self._ft_has_default)) - if exchange_config.get("_ft_has_params"): - self._ft_has = deep_merge_dicts(exchange_config.get("_ft_has_params"), + if exchange_config.get('_ft_has_params'): + self._ft_has = deep_merge_dicts(exchange_config.get('_ft_has_params'), self._ft_has) logger.info("Overriding exchange._ft_has with config params, result: %s", self._ft_has) @@ -115,14 +117,15 @@ class Exchange: if validate: # Check if timeframe is available - self.validate_timeframes(config.get('ticker_interval')) + self.validate_timeframes(config.get('timeframe')) # Initial markets load self._load_markets() # Check if all pairs are available self.validate_stakecurrency(config['stake_currency']) - self.validate_pairs(config['exchange']['pair_whitelist']) + if not exchange_config.get('skip_pair_validation'): + self.validate_pairs(config['exchange']['pair_whitelist']) self.validate_ordertypes(config.get('order_types', {})) self.validate_order_time_in_force(config.get('order_time_in_force', {})) self.validate_required_startup_candles(config.get('startup_candle_count', 0)) @@ -186,11 +189,16 @@ class Exchange: def timeframes(self) -> List[str]: return list((self._api.timeframes or {}).keys()) + @property + def ohlcv_candle_limit(self) -> int: + """exchange ohlcv candle limit""" + return int(self._ohlcv_candle_limit) + @property def markets(self) -> Dict: """exchange ccxt markets""" if not self._api.markets: - logger.warning("Markets were not loaded. Loading them now..") + logger.info("Markets were not loaded. Loading them now..") self._load_markets() return self._api.markets @@ -216,7 +224,7 @@ class Exchange: if quote_currencies: markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies} if pairs_only: - markets = {k: v for k, v in markets.items() if symbol_is_pair(v['symbol'])} + markets = {k: v for k, v in markets.items() if self.market_is_tradable(v)} if active_only: markets = {k: v for k, v in markets.items() if market_is_active(v)} return markets @@ -240,6 +248,19 @@ class Exchange: """ return self.markets.get(pair, {}).get('base', '') + def market_is_tradable(self, market: Dict[str, Any]) -> bool: + """ + Check if the market symbol is tradable by Freqtrade. + By default, checks if it's splittable by `/` and both sides correspond to base / quote + """ + symbol_parts = market['symbol'].split('/') + return (len(symbol_parts) == 2 and + len(symbol_parts[0]) > 0 and + len(symbol_parts[1]) > 0 and + symbol_parts[0] == market.get('base') and + symbol_parts[1] == market.get('quote') + ) + def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: if pair_interval in self._klines: return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] @@ -252,8 +273,8 @@ class Exchange: api.urls['api'] = api.urls['test'] logger.info("Enabled Sandbox API on %s", name) else: - logger.warning(name, "No Sandbox URL in CCXT, exiting. " - "Please check your config.json") + logger.warning( + f"No Sandbox URL in CCXT for {name}, exiting. Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') def _load_async_markets(self, reload: bool = False) -> None: @@ -262,7 +283,7 @@ class Exchange: asyncio.get_event_loop().run_until_complete( self._api_async.load_markets(reload=reload)) - except ccxt.BaseError as e: + except (asyncio.TimeoutError, ccxt.BaseError) as e: logger.warning('Could not load async markets. Reason: %s', e) return @@ -271,21 +292,23 @@ class Exchange: try: self._api.load_markets() self._load_async_markets() - self._last_markets_refresh = arrow.utcnow().timestamp + self._last_markets_refresh = arrow.utcnow().int_timestamp except ccxt.BaseError as e: logger.warning('Unable to initialize markets. Reason: %s', e) - def _reload_markets(self) -> None: - """Reload markets both sync and async, if refresh interval has passed""" + def reload_markets(self) -> None: + """Reload markets both sync and async if refresh interval has passed """ # Check whether markets have to be reloaded if (self._last_markets_refresh > 0) and ( self._last_markets_refresh + self.markets_refresh_interval - > arrow.utcnow().timestamp): + > arrow.utcnow().int_timestamp): return None logger.debug("Performing scheduled market reload..") try: self._api.load_markets(reload=True) - self._last_markets_refresh = arrow.utcnow().timestamp + # Also reload async markets to avoid issues with newly listed pairs + self._load_async_markets(reload=True) + self._last_markets_refresh = arrow.utcnow().int_timestamp except ccxt.BaseError: logger.exception("Could not reload markets.") @@ -349,7 +372,7 @@ class Exchange: for pair in [f"{curr_1}/{curr_2}", f"{curr_2}/{curr_1}"]: if pair in self.markets and self.markets[pair].get('active'): return pair - raise DependencyException(f"Could not combine {curr_1} and {curr_2} to get a valid pair.") + raise ExchangeError(f"Could not combine {curr_1} and {curr_2} to get a valid pair.") def validate_timeframes(self, timeframe: Optional[str]) -> None: """ @@ -466,18 +489,20 @@ class Exchange: def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict[str, Any]: - order_id = f'dry_run_{side}_{randint(0, 10**6)}' + order_id = f'dry_run_{side}_{datetime.now().timestamp()}' _amount = self.amount_to_precision(pair, amount) dry_order = { - "id": order_id, - 'pair': pair, + 'id': order_id, + 'symbol': pair, 'price': rate, + 'average': rate, 'amount': _amount, 'cost': _amount * rate, 'type': ordertype, 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), + 'timestamp': int(arrow.utcnow().int_timestamp * 1000), 'status': "closed" if ordertype == "market" else "open", 'fee': None, 'info': {} @@ -499,7 +524,7 @@ class Exchange: 'rate': self.get_fee(pair) } }) - if closed_order["type"] in ["stop_loss_limit"]: + if closed_order["type"] in ["stop_loss_limit", "stop-loss-limit"]: closed_order["info"].update({"stopPrice": closed_order["price"]}) self._dry_run_open_orders[closed_order["id"]] = closed_order @@ -516,15 +541,17 @@ class Exchange: amount, rate_for_order, params) except ccxt.InsufficientFunds as e: - raise DependencyException( - f'Insufficient funds to create {ordertype} {side} order on market {pair}.' + raise InsufficientFundsError( + f'Insufficient funds to create {ordertype} {side} order on market {pair}. ' f'Tried to {side} amount {amount} at rate {rate}.' f'Message: {e}') from e except ccxt.InvalidOrder as e: - raise DependencyException( - f'Could not create {ordertype} {side} order on market {pair}.' - f'Tried to {side} amount {amount} at rate {rate}.' + raise ExchangeError( + f'Could not create {ordertype} {side} order on market {pair}. ' + f'Tried to {side} amount {amount} at rate {rate}. ' f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not place {side} order due to {e.__class__.__name__}. Message: {e}') from e @@ -604,6 +631,8 @@ class Exchange: balances.pop("used", None) return balances + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e @@ -618,6 +647,8 @@ class Exchange: raise OperationalException( f'Exchange {self._api.name} does not support fetching tickers in batch. ' f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not load tickers due to {e.__class__.__name__}. Message: {e}') from e @@ -627,10 +658,13 @@ class Exchange: @retrier def fetch_ticker(self, pair: str) -> dict: try: - if pair not in self._api.markets or not self._api.markets[pair].get('active'): - raise DependencyException(f"Pair {pair} not available") + if (pair not in self._api.markets or + self._api.markets[pair].get('active', False) is False): + raise ExchangeError(f"Pair {pair} not available") data = self._api.fetch_ticker(pair) return data + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e @@ -646,15 +680,31 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from - :returns List with candle (OHLCV) data + :return: List with candle (OHLCV) data """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms)) + def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, + since_ms: int) -> DataFrame: + """ + Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe + :param pair: Pair to download + :param timeframe: Timeframe to get data for + :param since_ms: Timestamp in milliseconds to get history from + :return: OHLCV DataFrame + """ + ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) + return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: + """ + Download historic ohlcv + """ one_call = timeframe_to_msecs(timeframe) * self._ohlcv_candle_limit logger.debug( @@ -664,27 +714,36 @@ class Exchange: ) input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in - range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] + range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] results = await asyncio.gather(*input_coroutines, return_exceptions=True) # Combine gathered results data: List = [] - for p, timeframe, res in results: + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res if p == pair: - data.extend(res) + data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) logger.info("Downloaded data for %s with length %s.", pair, len(data)) return data - def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes) -> List[Tuple[str, List]]: + def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, + since_ms: Optional[int] = None, cache: bool = True + ) -> Dict[Tuple[str, str], DataFrame]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). Only used in the dataprovider.refresh() method. :param pair_list: List of 2 element tuples containing pair, interval to refresh - :return: TODO: return value is only used in the tests, get rid of it + :param since_ms: time since when to download, in milliseconds + :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists + :return: Dict of [{(pair, timeframe): Dataframe}] """ logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) @@ -694,7 +753,8 @@ class Exchange: for pair, timeframe in set(pair_list): if (not ((pair, timeframe) in self._klines) or self._now_is_time_to_refresh(pair, timeframe)): - input_coroutines.append(self._async_get_candle_history(pair, timeframe)) + input_coroutines.append(self._async_get_candle_history(pair, timeframe, + since_ms=since_ms)) else: logger.debug( "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", @@ -704,30 +764,32 @@ class Exchange: results = asyncio.get_event_loop().run_until_complete( asyncio.gather(*input_coroutines, return_exceptions=True)) + results_df = {} # handle caching for res in results: if isinstance(res, Exception): logger.warning("Async code raised an exception: %s", res.__class__.__name__) continue - pair = res[0] - timeframe = res[1] - ticks = res[2] + # Deconstruct tuple (has 3 elements) + pair, timeframe, ticks = res # keeping last candle time as last refreshed time of the pair if ticks: self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache - self._klines[(pair, timeframe)] = ohlcv_to_dataframe( - ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) - - return results + ohlcv_df = ohlcv_to_dataframe( + ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + results_df[(pair, timeframe)] = ohlcv_df + if cache: + self._klines[(pair, timeframe)] = ohlcv_df + return results_df def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool: # Timeframe in seconds interval_in_sec = timeframe_to_seconds(timeframe) return not ((self._pairs_last_refresh_time.get((pair, timeframe), 0) - + interval_in_sec) >= arrow.utcnow().timestamp) + + interval_in_sec) >= arrow.utcnow().int_timestamp) @retrier_async async def _async_get_candle_history(self, pair: str, timeframe: str, @@ -745,7 +807,8 @@ class Exchange: ) data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, - since=since_ms) + since=since_ms, + limit=self._ohlcv_candle_limit) # Some exchanges sort OHLCV in ASC order and others in DESC. # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) @@ -764,6 +827,8 @@ class Exchange: raise OperationalException( f'Exchange {self._api.name} does not support fetching historical ' f'candle (OHLCV) data. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError(f'Could not fetch historical candle (OHLCV) data ' f'for pair {pair} due to {e.__class__.__name__}. ' @@ -800,6 +865,8 @@ class Exchange: raise OperationalException( f'Exchange {self._api.name} does not support fetching historical trade data.' f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError(f'Could not load trade history due to {e.__class__.__name__}. ' f'Message: {e}') from e @@ -931,7 +998,7 @@ class Exchange: def check_order_canceled_empty(self, order: Dict) -> bool: """ Verify if an order has been cancelled without being partially filled - :param order: Order dict as returned from get_order() + :param order: Order dict as returned from fetch_order() :return: True if order has been cancelled without being filled, False otherwise. """ return order.get('status') in ('closed', 'canceled') and order.get('filled') == 0.0 @@ -939,19 +1006,29 @@ class Exchange: @retrier def cancel_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - return {} + order = self._dry_run_open_orders.get(order_id) + if order: + order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) + return order + else: + return {} try: return self._api.cancel_order(order_id, pair) except ccxt.InvalidOrder as e: raise InvalidOrderException( f'Could not cancel order. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + # Assign method to cancel_stoploss_order to allow easy overriding in other classes + cancel_stoploss_order = cancel_order + def is_cancel_order_result_suitable(self, corder) -> bool: if not isinstance(corder, dict): return False @@ -963,7 +1040,7 @@ class Exchange: """ Cancel order returning a result. Creates a fake result if cancel order returns a non-usable result - and get_order does not work (certain exchanges don't return cancelled orders) + and fetch_order does not work (certain exchanges don't return cancelled orders) :param order_id: Orderid to cancel :param pair: Pair corresponding to order_id :param amount: Amount to use for fake response @@ -974,17 +1051,17 @@ class Exchange: if self.is_cancel_order_result_suitable(corder): return corder except InvalidOrderException: - logger.warning(f"Could not cancel order {order_id}.") + logger.warning(f"Could not cancel order {order_id} for {pair}.") try: - order = self.get_order(order_id, pair) + order = self.fetch_order(order_id, pair) except InvalidOrderException: logger.warning(f"Could not fetch cancelled order {order_id}.") order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} return order - @retrier - def get_order(self, order_id: str, pair: str) -> Dict: + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) + def fetch_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: try: order = self._dry_run_open_orders[order_id] @@ -995,30 +1072,62 @@ class Exchange: f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e try: return self._api.fetch_order(order_id, pair) + except ccxt.OrderNotFound as e: + raise RetryableOrderError( + f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e except ccxt.InvalidOrder as e: raise InvalidOrderException( - f'Tried to get an invalid order (id: {order_id}). Message: {e}') from e + f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + # Assign method to fetch_stoploss_order to allow easy overriding in other classes + fetch_stoploss_order = fetch_order + + def fetch_order_or_stoploss_order(self, order_id: str, pair: str, + stoploss_order: bool = False) -> Dict: + """ + Simple wrapper calling either fetch_order or fetch_stoploss_order depending on + the stoploss_order parameter + :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. + """ + if stoploss_order: + return self.fetch_stoploss_order(order_id, pair) + return self.fetch_order(order_id, pair) + + @staticmethod + def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]]): + """ + Get next greater value in the list. + Used by fetch_l2_order_book if the api only supports a limited range + """ + if not limit_range: + return limit + return min([x for x in limit_range if limit <= x] + [max(limit_range)]) + @retrier def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: """ - get order book level 2 from exchange - - Notes: - 20180619: bittrex doesnt support limits -.- + Get L2 order book from exchange. + Can be limited to a certain amount (if supported). + Returns a dict in the format + {'asks': [price, volume], 'bids': [price, volume]} """ + limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range']) try: - return self._api.fetch_l2_order_book(pair, limit) + return self._api.fetch_l2_order_book(pair, limit1) except ccxt.NotSupported as e: raise OperationalException( f'Exchange {self._api.name} does not support fetching order book.' f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e @@ -1055,7 +1164,8 @@ class Exchange: matched_trades = [trade for trade in my_trades if trade['order'] == order_id] return matched_trades - + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e @@ -1072,6 +1182,8 @@ class Exchange: return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, price=price, takerOrMaker=taker_or_maker)['rate'] + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e @@ -1106,7 +1218,7 @@ class Exchange: if fee_curr in self.get_pair_base_currency(order['symbol']): # Base currency - divide by amount return round( - order['fee']['cost'] / safe_value_fallback(order, order, 'filled', 'amount'), 8) + order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8) elif fee_curr in self.get_pair_quote_currency(order['symbol']): # Quote currency - divide by cost return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None @@ -1119,9 +1231,9 @@ class Exchange: comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) tick = self.fetch_ticker(comb) - fee_to_quote_rate = safe_value_fallback(tick, tick, 'last', 'ask') + fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) - except DependencyException: + except ExchangeError: return None def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: @@ -1134,7 +1246,6 @@ class Exchange: return (order['fee']['cost'], order['fee']['currency'], self.calculate_fee_rate(order)) - # calculate rate ? (order['fee']['cost'] / (order['amount'] * order['price'])) def is_exchange_bad(exchange_name: str) -> bool: @@ -1220,20 +1331,6 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) -def symbol_is_pair(market_symbol: str, base_currency: str = None, - quote_currency: str = None) -> bool: - """ - Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the - quote currency separated by '/' character. If base_currency and/or quote_currency is passed, - it also checks that the symbol contains appropriate base and/or quote currency part before - and after the separating character correspondingly. - """ - symbol_parts = market_symbol.split('/') - return (len(symbol_parts) == 2 and - (symbol_parts[0] == base_currency if base_currency else len(symbol_parts[0]) > 0) and - (symbol_parts[1] == quote_currency if quote_currency else len(symbol_parts[1]) > 0)) - - def market_is_active(market: Dict) -> bool: """ Return True if the market is active. diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 75915122b..f05490cbb 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -1,8 +1,14 @@ """ FTX exchange subclass """ import logging -from typing import Dict +from typing import Any, Dict +import ccxt + +from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange import Exchange +from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier + logger = logging.getLogger(__name__) @@ -10,5 +16,121 @@ logger = logging.getLogger(__name__) class Ftx(Exchange): _ft_has: Dict = { + "stoploss_on_exchange": True, "ohlcv_candle_limit": 1500, } + + def market_is_tradable(self, market: Dict[str, Any]) -> bool: + """ + Check if the market symbol is tradable by Freqtrade. + Default checks + check if pair is spot pair (no futures trading yet). + """ + parent_check = super().market_is_tradable(market) + + return (parent_check and + market.get('spot', False) is True) + + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + return order['type'] == 'stop' and stop_loss > float(order['price']) + + @retrier(retries=0) + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: + """ + Creates a stoploss order. + depending on order_types.stoploss configuration, uses 'market' or limit order. + + Limit orders are defined by having orderPrice set, otherwise a market order is used. + """ + limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) + limit_rate = stop_price * limit_price_pct + + ordertype = "stop" + + stop_price = self.price_to_precision(pair, stop_price) + + if self._config['dry_run']: + dry_order = self.dry_run_order( + pair, ordertype, "sell", amount, stop_price) + return dry_order + + try: + params = self._params.copy() + if order_types.get('stoploss', 'market') == 'limit': + # set orderPrice to place limit order, otherwise it's a market order + params['orderPrice'] = limit_rate + + amount = self.amount_to_precision(pair, amount) + + order = self._api.create_order(symbol=pair, type=ordertype, side='sell', + amount=amount, price=stop_price, params=params) + logger.info('stoploss order added for %s. ' + 'stop price: %s.', pair, stop_price) + return order + except ccxt.InsufficientFunds as e: + raise InsufficientFundsError( + f'Insufficient funds to create {ordertype} sell order on market {pair}. ' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) + def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + try: + order = self._dry_run_open_orders[order_id] + return order + except KeyError as e: + # Gracefully handle errors with dry-run orders. + raise InvalidOrderException( + f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + try: + orders = self._api.fetch_orders(pair, None, params={'type': 'stop'}) + + order = [order for order in orders if order['id'] == order_id] + if len(order) == 1: + return order[0] + else: + raise InvalidOrderException(f"Could not get stoploss order for id {order_id}") + + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Tried to get an invalid order (id: {order_id}). Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @retrier + def cancel_stoploss_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + return {} + try: + return self._api.cancel_order(order_id, pair, params={'type': 'stop'}) + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not cancel order. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 932d82a27..6dbb751e5 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -1,14 +1,15 @@ """ Kraken exchange subclass """ import logging -from typing import Dict +from typing import Any, Dict import ccxt -from freqtrade.exceptions import (DependencyException, InvalidOrderException, +from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Exchange from freqtrade.exchange.common import retrier + logger = logging.getLogger(__name__) @@ -17,10 +18,21 @@ class Kraken(Exchange): _params: Dict = {"trading_agreement": "agree"} _ft_has: Dict = { "stoploss_on_exchange": True, + "ohlcv_candle_limit": 720, "trades_pagination": "id", "trades_pagination_arg": "since", } + def market_is_tradable(self, market: Dict[str, Any]) -> bool: + """ + Check if the market symbol is tradable by Freqtrade. + Default checks + check if pair is darkpool pair. + """ + parent_check = super().market_is_tradable(market) + + return (parent_check and + market.get('darkpool', False) is False) + @retrier def get_balances(self) -> dict: if self._config['dry_run']: @@ -45,6 +57,8 @@ class Kraken(Exchange): balances[bal]['free'] = balances[bal]['total'] - balances[bal]['used'] return balances + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e @@ -56,15 +70,24 @@ class Kraken(Exchange): Verify stop_loss against stoploss-order value (limit or price) Returns True if adjustment is necessary. """ - return order['type'] == 'stop-loss' and stop_loss > float(order['price']) + return (order['type'] in ('stop-loss', 'stop-loss-limit') + and stop_loss > float(order['price'])) + @retrier(retries=0) def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: """ Creates a stoploss market order. Stoploss market orders is the only stoploss type supported by kraken. """ + params = self._params.copy() - ordertype = "stop-loss" + if order_types.get('stoploss', 'market') == 'limit': + ordertype = "stop-loss-limit" + limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) + limit_rate = stop_price * limit_price_pct + params['price2'] = self.price_to_precision(pair, limit_rate) + else: + ordertype = "stop-loss" stop_price = self.price_to_precision(pair, stop_price) @@ -74,8 +97,6 @@ class Kraken(Exchange): return dry_order try: - params = self._params.copy() - amount = self.amount_to_precision(pair, amount) order = self._api.create_order(symbol=pair, type=ordertype, side='sell', @@ -84,8 +105,8 @@ class Kraken(Exchange): 'stop price: %s.', pair, stop_price) return order except ccxt.InsufficientFunds as e: - raise DependencyException( - f'Insufficient funds to create {ordertype} sell order on market {pair}.' + raise InsufficientFundsError( + f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' f'Message: {e}') from e except ccxt.InvalidOrder as e: @@ -93,6 +114,8 @@ class Kraken(Exchange): f'Could not create {ordertype} sell order on market {pair}. ' f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5104e4f95..d60b111f2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -4,25 +4,27 @@ Freqtrade is the main module of this bot. It contains the class Freqtrade() import copy import logging import traceback -from datetime import datetime +from datetime import datetime, timezone from math import isclose from threading import Lock from typing import Any, Dict, List, Optional import arrow from cachetools import TTLCache -from requests.exceptions import RequestException -from freqtrade import __version__, constants, persistence +from freqtrade import __version__, constants from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.exceptions import DependencyException, InvalidOrderException, PricingError -from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date -from freqtrade.misc import safe_value_fallback -from freqtrade.pairlist.pairlistmanager import PairListManager -from freqtrade.persistence import Trade +from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, + InvalidOrderException, PricingError) +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds +from freqtrade.misc import safe_value_fallback, safe_value_fallback2 +from freqtrade.mixins import LoggingMixin +from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db +from freqtrade.plugins.pairlistmanager import PairListManager +from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State @@ -30,10 +32,11 @@ from freqtrade.strategy.interface import IStrategy, SellType from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets + logger = logging.getLogger(__name__) -class FreqtradeBot: +class FreqtradeBot(LoggingMixin): """ Freqtrade is the main class of the bot. This is from here the bot start its logic. @@ -57,8 +60,8 @@ class FreqtradeBot: # Cache values for 1800 to avoid frequent polling of the exchange for prices # Caching only applies to RPC methods, so prices for open trades are still # refreshed once every iteration. - self._sell_rate_cache = TTLCache(maxsize=100, ttl=1800) - self._buy_rate_cache = TTLCache(maxsize=100, ttl=1800) + self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) + self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) @@ -67,14 +70,18 @@ class FreqtradeBot: self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) - persistence.init(self.config.get('db_url', None), clean_open_orders=self.config['dry_run']) + init_db(self.config.get('db_url', None), clean_open_orders=self.config['dry_run']) self.wallets = Wallets(self.config, self.exchange) + PairLocks.timeframe = self.config['timeframe'] + self.pairlists = PairListManager(self.exchange, self.config) self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) + self.protections = ProtectionManager(self.config) + # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass @@ -98,6 +105,7 @@ class FreqtradeBot: self.rpc: RPCManager = RPCManager(self) # Protect sell-logic from forcesell and viceversa self._sell_lock = Lock() + LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) def notify_status(self, msg: str) -> None: """ @@ -119,19 +127,25 @@ class FreqtradeBot: if self.config['cancel_open_orders_on_exit']: self.cancel_all_open_orders() + self.check_for_open_trades() + self.rpc.cleanup() - persistence.cleanup() + cleanup_db() def startup(self) -> None: """ Called on startup and after reloading the bot - triggers notifications and performs startup tasks """ - self.rpc.startup_messages(self.config, self.pairlists) + self.rpc.startup_messages(self.config, self.pairlists, self.protections) if not self.edge: # Adjust stoploss if it was changed Trade.stoploss_reinitialization(self.strategy.stoploss) + # Only update open orders on startup + # This will update the database after the initial migration + self.update_open_orders() + def process(self) -> None: """ Queries the persistence layer for open trades and handles them, @@ -139,8 +153,10 @@ class FreqtradeBot: :return: True if one or more trades has been created or closed, False otherwise """ - # Check whether markets have to be reloaded - self.exchange._reload_markets() + # Check whether markets have to be reloaded and reload them when it's needed + self.exchange.reload_markets() + + self.update_closed_trades_without_assigned_fees() # Query trades from persistence layer trades = Trade.get_open_trades() @@ -151,6 +167,10 @@ class FreqtradeBot: self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist), self.strategy.informative_pairs()) + strategy_safe_wrapper(self.strategy.bot_loop_start, supress_error=True)() + + self.strategy.analyze(self.active_pair_whitelist) + with self._sell_lock: # Check and handle any timed out open orders self.check_handle_timedout() @@ -175,6 +195,24 @@ class FreqtradeBot: if self.config['cancel_open_orders_on_exit']: self.cancel_all_open_orders() + def check_for_open_trades(self): + """ + Notify the user when the bot is stopped + and there are still open trades active. + """ + open_trades = Trade.get_trades([Trade.is_open == 1]).all() + + if len(open_trades) != 0: + msg = { + 'type': RPCMessageType.WARNING_NOTIFICATION, + 'status': f"{len(open_trades)} open trades active.\n\n" + f"Handle these trades manually on {self.exchange.name}, " + f"or '/start' the bot again and use '/stopbuy' " + f"to handle open trades gracefully. \n" + f"{'Trades are simulated.' if self.config['dry_run'] else ''}", + } + self.rpc.send_msg(msg) + def _refresh_active_whitelist(self, trades: List[Trade] = []) -> List[str]: """ Refresh active whitelist from pairlist or edge and extend it with @@ -203,6 +241,104 @@ class FreqtradeBot: open_trades = len(Trade.get_open_trades()) return max(0, self.config['max_open_trades'] - open_trades) + def update_open_orders(self): + """ + Updates open orders based on order list kept in the database. + Mainly updates the state of orders - but may also close trades + """ + orders = Order.get_open_orders() + logger.info(f"Updating {len(orders)} open orders.") + for order in orders: + try: + fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair, + order.ft_order_side == 'stoploss') + + self.update_trade_state(order.trade, order.order_id, fo) + + except ExchangeError as e: + logger.warning(f"Error updating Order {order.order_id} due to {e}") + + def update_closed_trades_without_assigned_fees(self): + """ + Update closed trades without close fees assigned. + Only acts when Orders are in the database, otherwise the last orderid is unknown. + """ + trades: List[Trade] = Trade.get_sold_trades_without_assigned_fees() + for trade in trades: + + if not trade.is_open and not trade.fee_updated('sell'): + # Get sell fee + order = trade.select_order('sell', False) + if order: + logger.info(f"Updating sell-fee on trade {trade} for order {order.order_id}.") + self.update_trade_state(trade, order.order_id, + stoploss_order=order.ft_order_side == 'stoploss') + + trades: List[Trade] = Trade.get_open_trades_without_assigned_fees() + for trade in trades: + if trade.is_open and not trade.fee_updated('buy'): + order = trade.select_order('buy', False) + if order: + logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") + self.update_trade_state(trade, order.order_id) + + def handle_insufficient_funds(self, trade: Trade): + """ + Determine if we ever opened a sell order for this trade. + If not, try update buy fees - otherwise "refind" the open order we obviously lost. + """ + sell_order = trade.select_order('sell', None) + if sell_order: + self.refind_lost_order(trade) + else: + self.reupdate_buy_order_fees(trade) + + def reupdate_buy_order_fees(self, trade: Trade): + """ + Get buy order from database, and try to reupdate. + Handles trades where the initial fee-update did not work. + """ + logger.info(f"Trying to reupdate buy fees for {trade}") + order = trade.select_order('buy', False) + if order: + logger.info(f"Updating buy-fee on trade {trade} for order {order.order_id}.") + self.update_trade_state(trade, order.order_id) + + def refind_lost_order(self, trade): + """ + Try refinding a lost trade. + Only used when InsufficientFunds appears on sell orders (stoploss or sell). + Tries to walk the stored orders and sell them off eventually. + """ + logger.info(f"Trying to refind lost order for {trade}") + for order in trade.orders: + logger.info(f"Trying to refind {order}") + fo = None + if not order.ft_is_open: + logger.debug(f"Order {order} is no longer open.") + continue + if order.ft_order_side == 'buy': + # Skip buy side - this is handled by reupdate_buy_order_fees + continue + try: + fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair, + order.ft_order_side == 'stoploss') + if order.ft_order_side == 'stoploss': + if fo and fo['status'] == 'open': + # Assume this as the open stoploss order + trade.stoploss_order_id = order.order_id + elif order.ft_order_side == 'sell': + if fo and fo['status'] == 'open': + # Assume this as the open order + trade.open_order_id = order.order_id + if fo: + logger.info(f"Found {order} for trade {trade}.jj") + self.update_trade_state(trade, order.order_id, fo, + stoploss_order=order.ft_order_side == 'stoploss') + + except ExchangeError: + logger.warning(f"Error updating {order.order_id}.") + # # BUY / enter positions / open trades logic and methods # @@ -216,27 +352,35 @@ class FreqtradeBot: whitelist = copy.deepcopy(self.active_pair_whitelist) if not whitelist: logger.info("Active pair whitelist is empty.") - else: - # Remove pairs for currently opened trades from the whitelist - for trade in Trade.get_open_trades(): - if trade.pair in whitelist: - whitelist.remove(trade.pair) - logger.debug('Ignoring %s in pair whitelist', trade.pair) + return trades_created + # Remove pairs for currently opened trades from the whitelist + for trade in Trade.get_open_trades(): + if trade.pair in whitelist: + whitelist.remove(trade.pair) + logger.debug('Ignoring %s in pair whitelist', trade.pair) - if not whitelist: - logger.info("No currency pair in active pair whitelist, " - "but checking to sell open trades.") + if not whitelist: + logger.info("No currency pair in active pair whitelist, " + "but checking to sell open trades.") + return trades_created + if PairLocks.is_global_lock(): + lock = PairLocks.get_pair_longest_lock('*') + if lock: + self.log_once(f"Global pairlock active until " + f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}. " + "Not creating new trades.", logger.info) else: - # Create entity and execute trade for each pair from whitelist - for pair in whitelist: - try: - trades_created += self.create_trade(pair) - except DependencyException as exception: - logger.warning('Unable to create trade for %s: %s', pair, exception) + self.log_once("Global pairlock active. Not creating new trades.", logger.info) + return trades_created + # Create entity and execute trade for each pair from whitelist + for pair in whitelist: + try: + trades_created += self.create_trade(pair) + except DependencyException as exception: + logger.warning('Unable to create trade for %s: %s', pair, exception) - if not trades_created: - logger.debug("Found no buy signals for whitelisted currencies. " - "Trying again...") + if not trades_created: + logger.debug("Found no buy signals for whitelisted currencies. Trying again...") return trades_created @@ -251,7 +395,7 @@ class FreqtradeBot: rate = self._buy_rate_cache.get(pair) # Check if cache has been invalidated if rate: - logger.info(f"Using cached buy rate for {pair}.") + logger.debug(f"Using cached buy rate for {pair}.") return rate bid_strategy = self.config.get('bid_strategy', {}) @@ -388,8 +532,7 @@ class FreqtradeBot: # reserve some percent defined in config (5% default) + stoploss amount_reserve_percent = 1.0 - self.config.get('amount_reserve_percent', constants.DEFAULT_AMOUNT_RESERVE_PERCENT) - if self.strategy.stoploss is not None: - amount_reserve_percent += self.strategy.stoploss + amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) @@ -409,8 +552,16 @@ class FreqtradeBot: """ logger.debug(f"create_trade for pair {pair}") - if self.strategy.is_pair_locked(pair): - logger.info(f"Pair {pair} is currently locked.") + analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe) + nowtime = analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None + if self.strategy.is_pair_locked(pair, nowtime): + lock = PairLocks.get_pair_longest_lock(pair, nowtime) + if lock: + self.log_once(f"Pair {pair} is still locked until " + f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}.", + logger.info) + else: + self.log_once(f"Pair {pair} is still locked.", logger.info) return False # get_free_open_trades is checked before create_trade is called @@ -420,9 +571,7 @@ class FreqtradeBot: return False # running get_signal on historical data fetched - (buy, sell) = self.strategy.get_signal( - pair, self.strategy.ticker_interval, - self.dataprovider.ohlcv(pair, self.strategy.ticker_interval)) + (buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df) if buy and not sell: stake_amount = self.get_trade_stake_amount(pair) @@ -485,6 +634,9 @@ class FreqtradeBot: # Calculate price buy_limit_requested = self.get_buy_rate(pair, True) + if not buy_limit_requested: + raise PricingError('Could not determine buy price.') + min_stake_amount = self._get_min_pair_stake_amount(pair, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( @@ -495,14 +647,22 @@ class FreqtradeBot: amount = stake_amount / buy_limit_requested order_type = self.strategy.order_types['buy'] + if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( + pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested, + time_in_force=time_in_force): + logger.info(f"User requested abortion of buying {pair}") + return False + amount = self.exchange.amount_to_precision(pair, amount) order = self.exchange.buy(pair=pair, ordertype=order_type, amount=amount, rate=buy_limit_requested, time_in_force=time_in_force) + order_obj = Order.parse_from_ccxt_object(order, pair, 'buy') order_id = order['id'] order_status = order.get('status', None) # we assume the order is executed at the price requested buy_limit_filled_price = buy_limit_requested + amount_requested = amount if order_status == 'expired' or order_status == 'rejected': order_tif = self.strategy.order_time_in_force['buy'] @@ -523,15 +683,14 @@ class FreqtradeBot: order['filled'], order['amount'], order['remaining'] ) stake_amount = order['cost'] - amount = order['amount'] - buy_limit_filled_price = order['price'] - order_id = None + amount = safe_value_fallback(order, 'filled', 'amount') + buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') # in case of FOK the order may be filled immediately and fully elif order_status == 'closed': stake_amount = order['cost'] - amount = order['amount'] - buy_limit_filled_price = order['price'] + amount = safe_value_fallback(order, 'filled', 'amount') + buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') @@ -539,6 +698,7 @@ class FreqtradeBot: pair=pair, stake_amount=stake_amount, amount=amount, + amount_requested=amount_requested, fee_open=fee, fee_close=fee, open_rate=buy_limit_filled_price, @@ -547,12 +707,13 @@ class FreqtradeBot: exchange=self.exchange.id, open_order_id=order_id, strategy=self.strategy.get_strategy_name(), - ticker_interval=timeframe_to_minutes(self.config['ticker_interval']) + timeframe=timeframe_to_minutes(self.config['timeframe']) ) + trade.orders.append(order_obj) # Update fees if order is closed if order_status == 'closed': - self.update_trade_state(trade, order) + self.update_trade_state(trade, order_id, order) Trade.session.add(trade) Trade.session.flush() @@ -569,6 +730,7 @@ class FreqtradeBot: Sends rpc notification when a buy occured. """ msg = { + 'trade_id': trade.id, 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, @@ -585,13 +747,14 @@ class FreqtradeBot: # Send the message self.rpc.send_msg(msg) - def _notify_buy_cancel(self, trade: Trade, order_type: str) -> None: + def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a buy cancel occured. """ current_rate = self.get_buy_rate(trade.pair, False) msg = { + 'trade_id': trade.id, 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, 'exchange': self.exchange.name.capitalize(), 'pair': trade.pair, @@ -603,6 +766,7 @@ class FreqtradeBot: 'amount': trade.amount, 'open_date': trade.open_date, 'current_rate': current_rate, + 'reason': reason, } # Send the message @@ -629,7 +793,7 @@ class FreqtradeBot: trades_closed += 1 except DependencyException as exception: - logger.warning('Unable to sell trade: %s', exception) + logger.warning('Unable to sell trade %s: %s', trade.pair, exception) # Updating wallets if any trade occured if trades_closed: @@ -660,7 +824,7 @@ class FreqtradeBot: rate = self._sell_rate_cache.get(pair) # Check if cache has been invalidated if rate: - logger.info(f"Using cached sell rate for {pair}.") + logger.debug(f"Using cached sell rate for {pair}.") return rate ask_strategy = self.config.get('ask_strategy', {}) @@ -697,9 +861,10 @@ class FreqtradeBot: if (config_ask_strategy.get('use_sell_signal', True) or config_ask_strategy.get('ignore_roi_if_buy_signal', False)): - (buy, sell) = self.strategy.get_signal( - trade.pair, self.strategy.ticker_interval, - self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval)) + analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair, + self.strategy.timeframe) + + (buy, sell) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df) if config_ask_strategy.get('use_order_book', False): order_book_min = config_ask_strategy.get('order_book_min', 1) @@ -736,7 +901,7 @@ class FreqtradeBot: logger.debug('Found no sell signal for %s.', trade) return False - def create_stoploss_order(self, trade: Trade, stop_price: float, rate: float) -> bool: + def create_stoploss_order(self, trade: Trade, stop_price: float) -> bool: """ Abstracts creating stoploss orders from the logic. Handles errors and updates the trade database object. @@ -747,15 +912,23 @@ class FreqtradeBot: stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount, stop_price=stop_price, order_types=self.strategy.order_types) + + order_obj = Order.parse_from_ccxt_object(stoploss_order, trade.pair, 'stoploss') + trade.orders.append(order_obj) trade.stoploss_order_id = str(stoploss_order['id']) return True + except InsufficientFundsError as e: + logger.warning(f"Unable to place stoploss order {e}.") + # Try to figure out what went wrong + self.handle_insufficient_funds(trade) + except InvalidOrderException as e: trade.stoploss_order_id = None logger.error(f'Unable to place a stoploss order on exchange. {e}') logger.warning('Selling the trade forcefully') self.execute_sell(trade, trade.stop_loss, sell_reason=SellType.EMERGENCY_SELL) - except DependencyException: + except ExchangeError: trade.stoploss_order_id = None logger.exception('Unable to place a stoploss order on exchange.') return False @@ -773,18 +946,22 @@ class FreqtradeBot: try: # First we check if there is already a stoploss on exchange - stoploss_order = self.exchange.get_order(trade.stoploss_order_id, trade.pair) \ - if trade.stoploss_order_id else None + stoploss_order = self.exchange.fetch_stoploss_order( + trade.stoploss_order_id, trade.pair) if trade.stoploss_order_id else None except InvalidOrderException as exception: logger.warning('Unable to fetch stoploss order: %s', exception) + if stoploss_order: + trade.update_order(stoploss_order) + # We check if stoploss order is fulfilled - if stoploss_order and stoploss_order['status'] == 'closed': + if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'): trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value - self.update_trade_state(trade, stoploss_order, sl_order=True) + self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order, + stoploss_order=True) # Lock pair for one candle to prevent immediate rebuys - self.strategy.lock_pair(trade.pair, - timeframe_to_next_date(self.config['ticker_interval'])) + self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), + reason='Auto lock') self._notify_sell(trade, "stoploss") return True @@ -795,20 +972,17 @@ class FreqtradeBot: return False # If buy order is fulfilled but there is no stoploss, we add a stoploss on exchange - if (not stoploss_order): - + if not stoploss_order: stoploss = self.edge.stoploss(pair=trade.pair) if self.edge else self.strategy.stoploss - stop_price = trade.open_rate * (1 + stoploss) - if self.create_stoploss_order(trade=trade, stop_price=stop_price, rate=stop_price): - trade.stoploss_last_update = datetime.now() + if self.create_stoploss_order(trade=trade, stop_price=stop_price): + trade.stoploss_last_update = datetime.utcnow() return False # If stoploss order is canceled for some reason we add it - if stoploss_order and stoploss_order['status'] == 'canceled': - if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss, - rate=trade.stop_loss): + if stoploss_order and stoploss_order['status'] in ('canceled', 'cancelled'): + if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss): return False else: trade.stoploss_order_id = None @@ -836,17 +1010,17 @@ class FreqtradeBot: update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: # cancelling the current stoploss on exchange first - logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s}) ' - 'in order to add another one ...', order['id']) + logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} " + f"(orderid:{order['id']}) in order to add another one ...") try: - self.exchange.cancel_order(order['id'], trade.pair) + co = self.exchange.cancel_stoploss_order(order['id'], trade.pair) + trade.update_order(co) except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {order['id']} " f"for pair {trade.pair}") # Create new stoploss order - if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss, - rate=trade.stop_loss): + if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") @@ -890,12 +1064,12 @@ class FreqtradeBot: try: if not trade.open_order_id: continue - order = self.exchange.get_order(trade.open_order_id, trade.pair) - except (RequestException, DependencyException, InvalidOrderException): + order = self.exchange.fetch_order(trade.open_order_id, trade.pair) + except (ExchangeError): logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) continue - fully_cancelled = self.update_trade_state(trade, order) + fully_cancelled = self.update_trade_state(trade, trade.open_order_id, order) if (order['side'] == 'buy' and (order['status'] == 'open' or fully_cancelled) and ( fully_cancelled @@ -923,8 +1097,8 @@ class FreqtradeBot: for trade in Trade.get_open_order_trades(): try: - order = self.exchange.get_order(trade.open_order_id, trade.pair) - except (DependencyException, InvalidOrderException): + order = self.exchange.fetch_order(trade.open_order_id, trade.pair) + except (ExchangeError): logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) continue @@ -943,9 +1117,14 @@ class FreqtradeBot: # Cancelled orders may have the status of 'canceled' or 'closed' if order['status'] not in ('canceled', 'closed'): - reason = constants.CANCEL_REASON['TIMEOUT'] corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair, trade.amount) + # Avoid race condition where the order could not be cancelled coz its already filled. + # Simply bailing here is the only safe way - as this order will then be + # handled in the next iteration. + if corder.get('status') not in ('canceled', 'closed'): + logger.warning(f"Order {trade.open_order_id} for {trade.pair} not cancelled.") + return False else: # Order was cancelled already, so we can reuse the existing dict corder = order @@ -954,14 +1133,13 @@ class FreqtradeBot: logger.info('Buy order %s for %s.', reason, trade) # Using filled to determine the filled amount - filled_amount = safe_value_fallback(corder, order, 'filled', 'filled') - + filled_amount = safe_value_fallback2(corder, order, 'filled', 'filled') if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): logger.info('Buy order fully cancelled. Removing %s from database.', trade) # if trade is not partially completed, just delete the trade - Trade.session.delete(trade) - Trade.session.flush() + trade.delete() was_trade_fully_canceled = True + reason += f", {constants.CANCEL_REASON['FULLY_CANCELLED']}" else: # if trade is partially complete, edit the stake details for the trade # and close the order @@ -970,17 +1148,15 @@ class FreqtradeBot: # we need to fall back to the values from order if corder does not contain these keys. trade.amount = filled_amount trade.stake_amount = trade.amount * trade.open_rate - self.update_trade_state(trade, corder, trade.amount) + self.update_trade_state(trade, trade.open_order_id, corder) trade.open_order_id = None logger.info('Partial buy order timeout for %s.', trade) - self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Remaining buy order for {trade.pair} cancelled due to timeout' - }) + reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}" self.wallets.update() - self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy']) + self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'], + reason=reason) return was_trade_fully_canceled def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str: @@ -1011,7 +1187,7 @@ class FreqtradeBot: trade.open_order_id = None else: # TODO: figure out how to handle partially complete sell orders - reason = constants.CANCEL_REASON['PARTIALLY_FILLED'] + reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] self.wallets.update() self._notify_sell_cancel( @@ -1067,7 +1243,7 @@ class FreqtradeBot: # First cancelling stoploss on exchange ... if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id: try: - self.exchange.cancel_order(trade.stoploss_order_id, trade.pair) + self.exchange.cancel_stoploss_order(trade.stoploss_order_id, trade.pair) except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}") @@ -1077,24 +1253,42 @@ class FreqtradeBot: order_type = self.strategy.order_types.get("emergencysell", "market") amount = self._safe_sell_amount(trade.pair, trade.amount) + time_in_force = self.strategy.order_time_in_force['sell'] - # Execute sell and update trade record - order = self.exchange.sell(pair=str(trade.pair), - ordertype=order_type, - amount=amount, rate=limit, - time_in_force=self.strategy.order_time_in_force['sell'] - ) + if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( + pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, + time_in_force=time_in_force, + sell_reason=sell_reason.value): + logger.info(f"User requested abortion of selling {trade.pair}") + return False + + try: + # Execute sell and update trade record + order = self.exchange.sell(pair=trade.pair, + ordertype=order_type, + amount=amount, rate=limit, + time_in_force=time_in_force + ) + except InsufficientFundsError as e: + logger.warning(f"Unable to place order {e}.") + # Try to figure out what went wrong + self.handle_insufficient_funds(trade) + return False + + order_obj = Order.parse_from_ccxt_object(order, trade.pair, 'sell') + trade.orders.append(order_obj) trade.open_order_id = order['id'] trade.close_rate_requested = limit trade.sell_reason = sell_reason.value # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': - self.update_trade_state(trade, order) + self.update_trade_state(trade, trade.open_order_id, order) Trade.session.flush() # Lock pair for one candle to prevent immediate rebuys - self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['ticker_interval'])) + self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), + reason='Auto lock') self._notify_sell(trade, order_type) @@ -1113,6 +1307,7 @@ class FreqtradeBot: msg = { 'type': RPCMessageType.SELL_NOTIFICATION, + 'trade_id': trade.id, 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'gain': gain, @@ -1155,6 +1350,7 @@ class FreqtradeBot: msg = { 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'trade_id': trade.id, 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'gain': gain, @@ -1185,34 +1381,40 @@ class FreqtradeBot: # Common update trade state methods # - def update_trade_state(self, trade: Trade, action_order: dict = None, - order_amount: float = None, sl_order: bool = False) -> bool: + def update_trade_state(self, trade: Trade, order_id: str, action_order: Dict[str, Any] = None, + stoploss_order: bool = False) -> bool: """ Checks trades with open orders and updates the amount if necessary Handles closing both buy and sell orders. + :param trade: Trade object of the trade we're analyzing + :param order_id: Order-id of the order we're analyzing + :param action_order: Already aquired order object :return: True if order has been cancelled without being filled partially, False otherwise """ - # Get order details for actual price per unit - if trade.open_order_id: - order_id = trade.open_order_id - elif trade.stoploss_order_id and sl_order: - order_id = trade.stoploss_order_id - else: + if not order_id: + logger.warning(f'Orderid for trade {trade} is empty.') return False + # Update trade with order values logger.info('Found open order for %s', trade) try: - order = action_order or self.exchange.get_order(order_id, trade.pair) + order = action_order or self.exchange.fetch_order_or_stoploss_order(order_id, + trade.pair, + stoploss_order) except InvalidOrderException as exception: logger.warning('Unable to fetch order %s: %s', order_id, exception) return False + + trade.update_order(order) + # Try update amount (binance-fix) try: - new_amount = self.get_real_amount(trade, order, order_amount) - if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): + new_amount = self.get_real_amount(trade, order) + if not isclose(safe_value_fallback(order, 'filled', 'amount'), new_amount, + abs_tol=constants.MATH_CLOSE_PREC): order['amount'] = new_amount order.pop('filled', None) - trade.recalc_open_trade_price() + trade.recalc_open_trade_value() except DependencyException as exception: logger.warning("Could not update trade amount: %s", exception) @@ -1224,6 +1426,8 @@ class FreqtradeBot: # Updating wallets when order is closed if not trade.is_open: + self.protections.stop_per_pair(trade.pair) + self.protections.global_stop() self.wallets.update() return False @@ -1245,7 +1449,7 @@ class FreqtradeBot: return real_amount return amount - def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: + def get_real_amount(self, trade: Trade, order: Dict) -> float: """ Detect and update trade fee. Calls trade.update_fee() uppon correct detection. @@ -1254,8 +1458,7 @@ class FreqtradeBot: :return: identical (or new) amount for the trade """ # Init variables - if order_amount is None: - order_amount = order['amount'] + order_amount = safe_value_fallback(order, 'filled', 'amount') # Only run for closed orders if trade.fee_updated(order.get('side', '')) or order['status'] == 'open': return order_amount @@ -1266,20 +1469,23 @@ class FreqtradeBot: fee_cost, fee_currency, fee_rate = self.exchange.extract_cost_curr_rate(order) logger.info(f"Fee for Trade {trade} [{order.get('side')}]: " f"{fee_cost:.8g} {fee_currency} - rate: {fee_rate}") - - trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) - if trade_base_currency == fee_currency: - # Apply fee to amount - return self.apply_fee_conditional(trade, trade_base_currency, - amount=order_amount, fee_abs=fee_cost) - return order_amount + if fee_rate is None or fee_rate < 0.02: + # Reject all fees that report as > 2%. + # These are most likely caused by a parsing bug in ccxt + # due to multiple trades (https://github.com/ccxt/ccxt/issues/8025) + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) + if trade_base_currency == fee_currency: + # Apply fee to amount + return self.apply_fee_conditional(trade, trade_base_currency, + amount=order_amount, fee_abs=fee_cost) + return order_amount return self.fee_detection_from_trades(trade, order, order_amount) def fee_detection_from_trades(self, trade: Trade, order: Dict, order_amount: float) -> float: """ fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee. """ - trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, + trades = self.exchange.get_trades_for_order(order['id'], trade.pair, trade.open_date) if len(trades) == 0: diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index aa08ee8a7..fbb05d879 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -1,14 +1,18 @@ import logging import sys - from logging import Formatter -from logging.handlers import RotatingFileHandler, SysLogHandler -from typing import Any, Dict, List +from logging.handlers import BufferingHandler, RotatingFileHandler, SysLogHandler +from typing import Any, Dict from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) +LOGFORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + +# Initialize bufferhandler - will be used for /log endpoints +bufferHandler = BufferingHandler(1000) +bufferHandler.setFormatter(Formatter(LOGFORMAT)) def _set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None: @@ -33,17 +37,38 @@ def _set_loggers(verbosity: int = 0, api_verbosity: str = 'info') -> None: ) +def get_existing_handlers(handlertype): + """ + Returns Existing handler or None (if the handler has not yet been added to the root handlers). + """ + return next((h for h in logging.root.handlers if isinstance(h, handlertype)), None) + + +def setup_logging_pre() -> None: + """ + Early setup for logging. + Uses INFO loglevel and only the Streamhandler. + Early messages (before proper logging setup) will therefore only be sent to additional + logging handlers after the real initialization, because we don't know which + ones the user desires beforehand. + """ + logging.basicConfig( + level=logging.INFO, + format=LOGFORMAT, + handlers=[logging.StreamHandler(sys.stderr), bufferHandler] + ) + + def setup_logging(config: Dict[str, Any]) -> None: """ Process -v/--verbose, --logfile options """ # Log level verbosity = config['verbosity'] - - # Log to stderr - log_handlers: List[logging.Handler] = [logging.StreamHandler(sys.stderr)] + logging.root.addHandler(bufferHandler) logfile = config.get('logfile') + if logfile: s = logfile.split(':') if s[0] == 'syslog': @@ -53,33 +78,41 @@ def setup_logging(config: Dict[str, Any]) -> None: # config['logfilename']), which defaults to '/dev/log', applicable for most # of the systems. address = (s[1], int(s[2])) if len(s) > 2 else s[1] if len(s) > 1 else '/dev/log' - handler = SysLogHandler(address=address) + handler_sl = get_existing_handlers(SysLogHandler) + if handler_sl: + logging.root.removeHandler(handler_sl) + handler_sl = SysLogHandler(address=address) # No datetime field for logging into syslog, to allow syslog # to perform reduction of repeating messages if this is set in the # syslog config. The messages should be equal for this. - handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) - log_handlers.append(handler) + handler_sl.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) + logging.root.addHandler(handler_sl) elif s[0] == 'journald': try: from systemd.journal import JournaldLogHandler except ImportError: raise OperationalException("You need the systemd python package be installed in " "order to use logging to journald.") - handler = JournaldLogHandler() + handler_jd = get_existing_handlers(JournaldLogHandler) + if handler_jd: + logging.root.removeHandler(handler_jd) + handler_jd = JournaldLogHandler() # No datetime field for logging into journald, to allow syslog # to perform reduction of repeating messages if this is set in the # syslog config. The messages should be equal for this. - handler.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) - log_handlers.append(handler) + handler_jd.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) + logging.root.addHandler(handler_jd) else: - log_handlers.append(RotatingFileHandler(logfile, - maxBytes=1024 * 1024, # 1Mb - backupCount=10)) + handler_rf = get_existing_handlers(RotatingFileHandler) + if handler_rf: + logging.root.removeHandler(handler_rf) + handler_rf = RotatingFileHandler(logfile, + maxBytes=1024 * 1024 * 10, # 10Mb + backupCount=10) + handler_rf.setFormatter(Formatter(LOGFORMAT)) + logging.root.addHandler(handler_rf) - logging.basicConfig( - level=logging.INFO if verbosity < 1 else logging.DEBUG, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=log_handlers - ) + logging.root.setLevel(logging.INFO if verbosity < 1 else logging.DEBUG) _set_loggers(verbosity, config.get('api_server', {}).get('verbosity', 'info')) + logger.info('Verbosity set to %s', verbosity) diff --git a/freqtrade/main.py b/freqtrade/main.py index 08bdc5e32..84d4b24f8 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -3,18 +3,18 @@ Main Freqtrade bot script. Read the documentation to know what cli arguments you need. """ - -from freqtrade.exceptions import FreqtradeException, OperationalException -import sys -# check min. python version -if sys.version_info < (3, 6): - sys.exit("Freqtrade requires Python version >= 3.6") - -# flake8: noqa E402 import logging +import sys from typing import Any, List + +# check min. python version +if sys.version_info < (3, 7): + sys.exit("Freqtrade requires Python version >= 3.7") + from freqtrade.commands import Arguments +from freqtrade.exceptions import FreqtradeException, OperationalException +from freqtrade.loggers import setup_logging_pre logger = logging.getLogger('freqtrade') @@ -28,6 +28,7 @@ def main(sysargv: List[str] = None) -> None: return_code: Any = 1 try: + setup_logging_pre() arguments = Arguments(sysargv) args = arguments.get_parsed_arg() diff --git a/freqtrade/misc.py b/freqtrade/misc.py index ac6084eb7..359d0d0e4 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -12,6 +12,7 @@ from typing.io import IO import numpy as np import rapidjson + logger = logging.getLogger(__name__) @@ -41,7 +42,7 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: return dates.dt.to_pydatetime() -def file_dump_json(filename: Path, data: Any, is_zip: bool = False) -> None: +def file_dump_json(filename: Path, data: Any, is_zip: bool = False, log: bool = True) -> None: """ Dump JSON data into a file :param filename: file to create @@ -52,12 +53,14 @@ def file_dump_json(filename: Path, data: Any, is_zip: bool = False) -> None: if is_zip: if filename.suffix != '.gz': filename = filename.with_suffix('.gz') - logger.info(f'dumping json to "{filename}"') + if log: + logger.info(f'dumping json to "{filename}"') - with gzip.open(filename, 'w') as fp: - rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) + with gzip.open(filename, 'w') as fpz: + rapidjson.dump(data, fpz, default=str, number_mode=rapidjson.NM_NATIVE) else: - logger.info(f'dumping json to "{filename}"') + if log: + logger.info(f'dumping json to "{filename}"') with open(filename, 'w') as fp: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) @@ -134,7 +137,21 @@ def round_dict(d, n): return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()} -def safe_value_fallback(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None): +def safe_value_fallback(obj: dict, key1: str, key2: str, default_value=None): + """ + Search a value in obj, return this if it's not None. + Then search key2 in obj - return that if it's not none - then use default_value. + Else falls back to None. + """ + if key1 in obj and obj[key1] is not None: + return obj[key1] + else: + if key2 in obj and obj[key2] is not None: + return obj[key2] + return default_value + + +def safe_value_fallback2(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None): """ Search a value in dict1, return this if it's not None. Fall back to dict2 - return key2 from dict2 if it's not None. diff --git a/freqtrade/mixins/__init__.py b/freqtrade/mixins/__init__.py new file mode 100644 index 000000000..f4a640fa3 --- /dev/null +++ b/freqtrade/mixins/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa: F401 +from freqtrade.mixins.logging_mixin import LoggingMixin diff --git a/freqtrade/mixins/logging_mixin.py b/freqtrade/mixins/logging_mixin.py new file mode 100644 index 000000000..06935d5f6 --- /dev/null +++ b/freqtrade/mixins/logging_mixin.py @@ -0,0 +1,38 @@ +from typing import Callable + +from cachetools import TTLCache, cached + + +class LoggingMixin(): + """ + Logging Mixin + Shows similar messages only once every `refresh_period`. + """ + # Disable output completely + show_output = True + + def __init__(self, logger, refresh_period: int = 3600): + """ + :param refresh_period: in seconds - Show identical messages in this intervals + """ + self.logger = logger + self.refresh_period = refresh_period + self._log_cache: TTLCache = TTLCache(maxsize=1024, ttl=self.refresh_period) + + def log_once(self, message: str, logmethod: Callable) -> None: + """ + Logs message - not more often than "refresh_period" to avoid log spamming + Logs the log-message as debug as well to simplify debugging. + :param message: String containing the message to be sent to the function. + :param logmethod: Function that'll be called. Most likely `logger.info`. + :return: None. + """ + @cached(cache=self._log_cache) + def _log_once(message: str): + logmethod(message) + + # Log as debug first + self.logger.debug(message) + # Call hidden function. + if self.show_output: + _log_once(message) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index b47b38ea4..a689786ec 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -4,31 +4,41 @@ This module contains the backtesting logic """ import logging +from collections import defaultdict from copy import deepcopy from datetime import datetime, timedelta from typing import Any, Dict, List, NamedTuple, Optional, Tuple -import arrow from pandas import DataFrame -from freqtrade.configuration import (TimeRange, remove_credentials, - validate_config_consistency) +from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency +from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data import history from freqtrade.data.converter import trim_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.optimize.optimize_reports import (generate_backtest_stats, - show_backtest_results, - store_backtest_result) -from freqtrade.pairlist.pairlistmanager import PairListManager -from freqtrade.persistence import Trade +from freqtrade.mixins import LoggingMixin +from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, + store_backtest_stats) +from freqtrade.persistence import PairLocks, Trade +from freqtrade.plugins.pairlistmanager import PairListManager +from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.state import RunMode from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType + logger = logging.getLogger(__name__) +# Indexes for backtest tuples +DATE_IDX = 0 +BUY_IDX = 1 +OPEN_IDX = 2 +CLOSE_IDX = 3 +SELL_IDX = 4 +LOW_IDX = 5 +HIGH_IDX = 6 + class BacktestResult(NamedTuple): """ @@ -37,14 +47,15 @@ class BacktestResult(NamedTuple): pair: str profit_percent: float profit_abs: float - open_time: datetime - close_time: datetime - open_index: int - close_index: int + open_date: datetime + open_rate: float + open_fee: float + close_date: datetime + close_rate: float + close_fee: float + amount: float trade_duration: float open_at_end: bool - open_rate: float - close_rate: float sell_reason: SellType @@ -58,6 +69,8 @@ class Backtesting: """ def __init__(self, config: Dict[str, Any]) -> None: + + LoggingMixin.show_output = False self.config = config # Reset keys for backtesting @@ -65,23 +78,8 @@ class Backtesting: self.strategylist: List[IStrategy] = [] self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) - self.pairlists = PairListManager(self.exchange, self.config) - if 'VolumePairList' in self.pairlists.name_list: - raise OperationalException("VolumePairList not allowed for backtesting.") - - self.pairlists.refresh_pairlist() - - if len(self.pairlists.whitelist) == 0: - raise OperationalException("No pair in whitelist.") - - if config.get('fee'): - self.fee = config['fee'] - else: - self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) - - if self.config.get('runmode') != RunMode.HYPEROPT: - self.dataprovider = DataProvider(self.config, self.exchange) - IStrategy.dp = self.dataprovider + dataprovider = DataProvider(self.config, self.exchange) + IStrategy.dp = dataprovider if self.config.get('strategy_list', None): for strat in list(self.config['strategy_list']): @@ -95,22 +93,57 @@ class Backtesting: self.strategylist.append(StrategyResolver.load_strategy(self.config)) validate_config_consistency(self.config) - if "ticker_interval" not in self.config: + if "timeframe" not in self.config: raise OperationalException("Timeframe (ticker interval) needs to be set in either " - "configuration or as cli argument `--ticker-interval 5m`") - self.timeframe = str(self.config.get('ticker_interval')) + "configuration or as cli argument `--timeframe 5m`") + self.timeframe = str(self.config.get('timeframe')) self.timeframe_min = timeframe_to_minutes(self.timeframe) + self.pairlists = PairListManager(self.exchange, self.config) + if 'VolumePairList' in self.pairlists.name_list: + raise OperationalException("VolumePairList not allowed for backtesting.") + if 'PerformanceFilter' in self.pairlists.name_list: + raise OperationalException("PerformanceFilter not allowed for backtesting.") + + if len(self.strategylist) > 1 and 'PrecisionFilter' in self.pairlists.name_list: + raise OperationalException( + "PrecisionFilter not allowed for backtesting multiple strategies." + ) + + dataprovider.add_pairlisthandler(self.pairlists) + self.pairlists.refresh_pairlist() + + if len(self.pairlists.whitelist) == 0: + raise OperationalException("No pair in whitelist.") + + if config.get('fee', None) is not None: + self.fee = config['fee'] + else: + self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) + + Trade.use_db = False + Trade.reset_trades() + PairLocks.timeframe = self.config['timeframe'] + PairLocks.use_db = False + PairLocks.reset_locks() + if self.config.get('enable_protections', False): + self.protections = ProtectionManager(self.config) + # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) # Load one (first) strategy self._set_strategy(self.strategylist[0]) + def __del__(self): + LoggingMixin.show_output = True + PairLocks.use_db = True + Trade.use_db = True + def _set_strategy(self, strategy): """ Load strategy into backtesting """ - self.strategy = strategy + self.strategy: IStrategy = strategy # Set stoploss_on_exchange to false for backtesting, # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case @@ -132,22 +165,35 @@ class Backtesting: min_date, max_date = history.get_timerange(data) - logger.info( - 'Loading data from %s up to %s (%s days)..', - min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days - ) + logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'({(max_date - min_date).days} days)..') + # Adjust startts forward if not enough data is available timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe), self.required_startup, min_date) return data, timerange - def _get_ohlcv_as_lists(self, processed: Dict) -> Dict[str, DataFrame]: + def prepare_backtest(self, enable_protections): + """ + Backtesting setup method - called once for every call to "backtest()". + """ + PairLocks.use_db = False + Trade.use_db = False + if enable_protections: + # Reset persisted data - used for protections only + PairLocks.reset_locks() + Trade.reset_trades() + + def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]: """ Helper function to convert a processed dataframes into lists for performance reasons. Used by backtest() - so keep this optimized for performance. """ + # Every change to this headers list must evaluate further usages of the resulting tuple + # and eventually change the constants for indexes at the top headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] data: Dict = {} # Create dict with data @@ -167,10 +213,10 @@ class Backtesting: # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) - data[pair] = [x for x in df_analyzed.itertuples()] + data[pair] = [x for x in df_analyzed.itertuples(index=False, name=None)] return data - def _get_close_rate(self, sell_row, trade: Trade, sell: SellCheckTuple, + def _get_close_rate(self, sell_row: Tuple, trade: Trade, sell: SellCheckTuple, trade_dur: int) -> float: """ Get close rate for backtesting result @@ -181,12 +227,12 @@ class Backtesting: return trade.stop_loss elif sell.sell_type == (SellType.ROI): roi_entry, roi = self.strategy.min_roi_reached_entry(trade_dur) - if roi is not None: + if roi is not None and roi_entry is not None: if roi == -1 and roi_entry % self.timeframe_min == 0: # When forceselling with ROI=-1, the roi time will always be equal to trade_dur. # If that entry is a multiple of the timeframe (so on candle open) # - we'll use open instead of close - return sell_row.open + return sell_row[OPEN_IDX] # - (Expected abs profit + open_rate + open_fee) / (fee_close -1) close_rate = - (trade.open_rate * roi + trade.open_rate * @@ -194,90 +240,86 @@ class Backtesting: if (trade_dur > 0 and trade_dur == roi_entry and roi_entry % self.timeframe_min == 0 - and sell_row.open > close_rate): + and sell_row[OPEN_IDX] > close_rate): # new ROI entry came into effect. # use Open rate if open_rate > calculated sell rate - return sell_row.open + return sell_row[OPEN_IDX] # Use the maximum between close_rate and low as we # cannot sell outside of a candle. # Applies when a new ROI setting comes in place and the whole candle is above that. - return max(close_rate, sell_row.low) + return max(close_rate, sell_row[LOW_IDX]) else: # This should not be reached... - return sell_row.open + return sell_row[OPEN_IDX] else: - return sell_row.open + return sell_row[OPEN_IDX] - def _get_sell_trade_entry( - self, pair: str, buy_row: DataFrame, - partial_ohlcv: List, trade_count_lock: Dict, - stake_amount: float, max_open_trades: int) -> Optional[BacktestResult]: + def _get_sell_trade_entry(self, trade: Trade, sell_row: Tuple) -> Optional[BacktestResult]: - trade = Trade( - pair=pair, - open_rate=buy_row.open, - open_date=buy_row.date, - stake_amount=stake_amount, - amount=stake_amount / buy_row.open, - fee_open=self.fee, - fee_close=self.fee, - is_open=True, - ) - logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.") - # calculate win/lose forwards from buy point - for sell_row in partial_ohlcv: - if max_open_trades > 0: - # Increase trade_count_lock for every iteration - trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 + sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], sell_row[DATE_IDX], + sell_row[BUY_IDX], sell_row[SELL_IDX], + low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX]) + if sell.sell_flag: + trade_dur = int((sell_row[DATE_IDX] - trade.open_date).total_seconds() // 60) + closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) - sell = self.strategy.should_sell(trade, sell_row.open, sell_row.date, sell_row.buy, - sell_row.sell, low=sell_row.low, high=sell_row.high) - if sell.sell_flag: - trade_dur = int((sell_row.date - buy_row.date).total_seconds() // 60) - closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) + trade.close_date = sell_row[DATE_IDX] + trade.sell_reason = sell.sell_type + trade.close(closerate, show_msg=False) - return BacktestResult(pair=pair, - profit_percent=trade.calc_profit_ratio(rate=closerate), - profit_abs=trade.calc_profit(rate=closerate), - open_time=buy_row.date, - close_time=sell_row.date, - trade_duration=trade_dur, - open_index=buy_row.Index, - close_index=sell_row.Index, - open_at_end=False, - open_rate=buy_row.open, - close_rate=closerate, - sell_reason=sell.sell_type - ) - if partial_ohlcv: - # no sell condition found - trade stil open at end of backtest period - sell_row = partial_ohlcv[-1] - bt_res = BacktestResult(pair=pair, - profit_percent=trade.calc_profit_ratio(rate=sell_row.open), - profit_abs=trade.calc_profit(rate=sell_row.open), - open_time=buy_row.date, - close_time=sell_row.date, - trade_duration=int(( - sell_row.date - buy_row.date).total_seconds() // 60), - open_index=buy_row.Index, - close_index=sell_row.Index, - open_at_end=True, - open_rate=buy_row.open, - close_rate=sell_row.open, - sell_reason=SellType.FORCE_SELL - ) - logger.debug(f"{pair} - Force selling still open trade, " - f"profit percent: {bt_res.profit_percent}, " - f"profit abs: {bt_res.profit_abs}") - - return bt_res + return BacktestResult(pair=trade.pair, + profit_percent=trade.calc_profit_ratio(rate=closerate), + profit_abs=trade.calc_profit(rate=closerate), + open_date=trade.open_date, + open_rate=trade.open_rate, + open_fee=self.fee, + close_date=sell_row[DATE_IDX], + close_rate=closerate, + close_fee=self.fee, + amount=trade.amount, + trade_duration=trade_dur, + open_at_end=False, + sell_reason=sell.sell_type + ) return None + def handle_left_open(self, open_trades: Dict[str, List[Trade]], + data: Dict[str, List[Tuple]]) -> List[BacktestResult]: + """ + Handling of left open trades at the end of backtesting + """ + trades = [] + for pair in open_trades.keys(): + if len(open_trades[pair]) > 0: + for trade in open_trades[pair]: + sell_row = data[pair][-1] + + trade_entry = BacktestResult(pair=trade.pair, + profit_percent=trade.calc_profit_ratio( + rate=sell_row[OPEN_IDX]), + profit_abs=trade.calc_profit(sell_row[OPEN_IDX]), + open_date=trade.open_date, + open_rate=trade.open_rate, + open_fee=self.fee, + close_date=sell_row[DATE_IDX], + close_rate=sell_row[OPEN_IDX], + close_fee=self.fee, + amount=trade.amount, + trade_duration=int(( + sell_row[DATE_IDX] - trade.open_date + ).total_seconds() // 60), + open_at_end=True, + sell_reason=SellType.FORCE_SELL + ) + trades.append(trade_entry) + return trades + def backtest(self, processed: Dict, stake_amount: float, - start_date: arrow.Arrow, end_date: arrow.Arrow, - max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame: + start_date: datetime, end_date: datetime, + max_open_trades: int = 0, position_stacking: bool = False, + enable_protections: bool = False) -> DataFrame: """ Implement backtesting functionality @@ -291,6 +333,7 @@ class Backtesting: :param end_date: backtesting timerange end datetime :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited :param position_stacking: do we allow position stacking? + :param enable_protections: Should protections be enabled? :return: DataFrame with trades (results of backtesting) """ logger.debug(f"Run backtest, stake_amount: {stake_amount}, " @@ -298,19 +341,22 @@ class Backtesting: f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}" ) trades = [] - trade_count_lock: Dict = {} + self.prepare_backtest(enable_protections) # Use dict of lists with data for performance # (looping lists is a lot faster than pandas DataFrames) data: Dict = self._get_ohlcv_as_lists(processed) - lock_pair_until: Dict = {} # Indexes per pair, so some pairs are allowed to have a missing start. indexes: Dict = {} tmp = start_date + timedelta(minutes=self.timeframe_min) + open_trades: Dict[str, List] = defaultdict(list) + open_trade_count = 0 + # Loop timerange and get candle for each pair at that point in time - while tmp < end_date: + while tmp <= end_date: + open_trade_count_start = open_trade_count for i, pair in enumerate(data): if pair not in indexes: @@ -324,42 +370,57 @@ class Backtesting: continue # Waits until the time-counter reaches the start of the data for this pair. - if row.date > tmp.datetime: + if row[DATE_IDX] > tmp: continue - indexes[pair] += 1 - if row.buy == 0 or row.sell == 1: - continue # skip rows where no buy signal or that would immediately sell off + # without positionstacking, we can only have one open trade per pair. + # max_open_trades must be respected + # don't open on the last row + if ((position_stacking or len(open_trades[pair]) == 0) + and (max_open_trades <= 0 or open_trade_count_start < max_open_trades) + and tmp != end_date + and row[BUY_IDX] == 1 and row[SELL_IDX] != 1 + and not PairLocks.is_pair_locked(pair, row[DATE_IDX])): + # Enter trade + trade = Trade( + pair=pair, + open_rate=row[OPEN_IDX], + open_date=row[DATE_IDX], + stake_amount=stake_amount, + amount=round(stake_amount / row[OPEN_IDX], 8), + fee_open=self.fee, + fee_close=self.fee, + is_open=True, + ) + # TODO: hacky workaround to avoid opening > max_open_trades + # This emulates previous behaviour - not sure if this is correct + # Prevents buying if the trade-slot was freed in this candle + open_trade_count_start += 1 + open_trade_count += 1 + # logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.") + open_trades[pair].append(trade) + Trade.trades.append(trade) - if (not position_stacking and pair in lock_pair_until - and row.date <= lock_pair_until[pair]): - # without positionstacking, we can only have one open trade per pair. - continue - - if max_open_trades > 0: - # Check if max_open_trades has already been reached for the given date - if not trade_count_lock.get(row.date, 0) < max_open_trades: - continue - trade_count_lock[row.date] = trade_count_lock.get(row.date, 0) + 1 - - # since indexes has been incremented before, we need to go one step back to - # also check the buying candle for sell conditions. - trade_entry = self._get_sell_trade_entry(pair, row, data[pair][indexes[pair]-1:], - trade_count_lock, stake_amount, - max_open_trades) - - if trade_entry: - logger.debug(f"{pair} - Locking pair till " - f"close_time={trade_entry.close_time}") - lock_pair_until[pair] = trade_entry.close_time - trades.append(trade_entry) - else: - # Set lock_pair_until to end of testing period if trade could not be closed - lock_pair_until[pair] = end_date.datetime + for trade in open_trades[pair]: + # since indexes has been incremented before, we need to go one step back to + # also check the buying candle for sell conditions. + trade_entry = self._get_sell_trade_entry(trade, row) + # Sell occured + if trade_entry: + # logger.debug(f"{pair} - Backtesting sell {trade}") + open_trade_count -= 1 + open_trades[pair].remove(trade) + trades.append(trade_entry) + if enable_protections: + self.protections.stop_per_pair(pair, row[DATE_IDX]) + self.protections.global_stop(tmp) # Move time one configured time_interval ahead. tmp += timedelta(minutes=self.timeframe_min) + + trades += self.handle_left_open(open_trades, data=data) + return DataFrame.from_records(trades, columns=BacktestResult._fields) def start(self) -> None: @@ -372,12 +433,6 @@ class Backtesting: logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) - # Use max_open_trades in backtesting, except --disable-max-market-positions is set - if self.config.get('use_max_market_positions', True): - max_open_trades = self.config['max_open_trades'] - else: - logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') - max_open_trades = 0 position_stacking = self.config.get('position_stacking', False) data, timerange = self.load_bt_data() @@ -387,6 +442,15 @@ class Backtesting: logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) self._set_strategy(strat) + # Use max_open_trades in backtesting, except --disable-max-market-positions is set + if self.config.get('use_max_market_positions', True): + # Must come from strategy config, as the strategy may modify this setting. + max_open_trades = self.strategy.config['max_open_trades'] + else: + logger.info( + 'Ignoring max_open_trades (--disable-max-market-positions was used) ...') + max_open_trades = 0 + # need to reprocess data every time to populate signals preprocessed = self.strategy.ohlcvdata_to_dataframe(data) @@ -395,22 +459,29 @@ class Backtesting: preprocessed[pair] = trim_dataframe(df, timerange) min_date, max_date = history.get_timerange(preprocessed) - logger.info( - 'Backtesting with data from %s up to %s (%s days)..', - min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days - ) + logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'({(max_date - min_date).days} days)..') # Execute backtest and print results - all_results[self.strategy.get_strategy_name()] = self.backtest( + results = self.backtest( processed=preprocessed, stake_amount=self.config['stake_amount'], - start_date=min_date, - end_date=max_date, + start_date=min_date.datetime, + end_date=max_date.datetime, max_open_trades=max_open_trades, position_stacking=position_stacking, + enable_protections=self.config.get('enable_protections', False), ) + all_results[self.strategy.get_strategy_name()] = { + 'results': results, + 'config': self.strategy.config, + 'locks': PairLocks.locks, + } + + stats = generate_backtest_stats(data, all_results, min_date=min_date, max_date=max_date) if self.config.get('export', False): - store_backtest_result(self.config['exportfilename'], all_results) + store_backtest_stats(self.config['exportfilename'], stats) + # Show backtest results - stats = generate_backtest_stats(self.config, data, all_results) show_backtest_results(self.config, stats) diff --git a/freqtrade/optimize/default_hyperopt_loss.py b/freqtrade/optimize/default_hyperopt_loss.py index 9e780d0ea..9dbdc4403 100644 --- a/freqtrade/optimize/default_hyperopt_loss.py +++ b/freqtrade/optimize/default_hyperopt_loss.py @@ -1,5 +1,5 @@ """ -DefaultHyperOptLoss +ShortTradeDurHyperOptLoss This module defines the default HyperoptLoss class which is being used for Hyperoptimization. """ @@ -26,7 +26,7 @@ EXPECTED_MAX_PROFIT = 3.0 MAX_ACCEPTED_TRADE_DURATION = 300 -class DefaultHyperOptLoss(IHyperOptLoss): +class ShortTradeDurHyperOptLoss(IHyperOptLoss): """ Defines the default loss function for hyperopt """ @@ -50,3 +50,7 @@ class DefaultHyperOptLoss(IHyperOptLoss): duration_loss = 0.4 * min(trade_duration / MAX_ACCEPTED_TRADE_DURATION, 1) result = trade_loss + profit_loss + duration_loss return result + + +# Create an alias for This to allow the legacy Method to work as well. +DefaultHyperOptLoss = ShortTradeDurHyperOptLoss diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index be19688d8..a5f505bee 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -7,12 +7,12 @@ import logging from typing import Any, Dict from freqtrade import constants -from freqtrade.configuration import (TimeRange, remove_credentials, - validate_config_consistency) +from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency from freqtrade.edge import Edge from freqtrade.optimize.optimize_reports import generate_edge_table from freqtrade.resolvers import ExchangeResolver, StrategyResolver + logger = logging.getLogger(__name__) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 3a28de785..2a2f5b472 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,37 +4,39 @@ This module contains the hyperopt logic """ +import io import locale import logging import random import warnings -from math import ceil from collections import OrderedDict +from datetime import datetime +from math import ceil from operator import itemgetter from pathlib import Path -from pprint import pprint +from pprint import pformat from typing import Any, Dict, List, Optional -import rapidjson -from colorama import Fore, Style -from joblib import (Parallel, cpu_count, delayed, dump, load, - wrap_non_picklable_objects) -from pandas import DataFrame, json_normalize, isna import progressbar +import rapidjson import tabulate -from os import path -import io +from colorama import Fore, Style +from colorama import init as colorama_init +from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects +from pandas import DataFrame, isna, json_normalize +from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframe from freqtrade.data.history import get_timerange from freqtrade.exceptions import OperationalException -from freqtrade.misc import plural, round_dict +from freqtrade.misc import file_dump_json, plural, round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 -from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, - HyperOptResolver) +from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver +from freqtrade.strategy import IStrategy + # Suppress scikit-learn FutureWarnings from skopt with warnings.catch_warnings(): @@ -74,19 +76,16 @@ class Hyperopt: self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config) self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function - + time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") self.results_file = (self.config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') + 'hyperopt_results' / f'hyperopt_results_{time_now}.pickle') self.data_pickle_file = (self.config['user_data_dir'] / 'hyperopt_results' / 'hyperopt_tickerdata.pkl') self.total_epochs = config.get('epochs', 0) self.current_best_loss = 100 - if not self.config.get('hyperopt_continue'): - self.clean_hyperopt() - else: - logger.info("Continuing on previous hyperopt results.") + self.clean_hyperopt() self.num_epochs_saved = 0 @@ -95,14 +94,14 @@ class Hyperopt: # Populate functions here (hasattr is slow so should not be run during "regular" operations) if hasattr(self.custom_hyperopt, 'populate_indicators'): - self.backtesting.strategy.advise_indicators = \ - self.custom_hyperopt.populate_indicators # type: ignore + self.backtesting.strategy.advise_indicators = ( # type: ignore + self.custom_hyperopt.populate_indicators) # type: ignore if hasattr(self.custom_hyperopt, 'populate_buy_trend'): - self.backtesting.strategy.advise_buy = \ - self.custom_hyperopt.populate_buy_trend # type: ignore + self.backtesting.strategy.advise_buy = ( # type: ignore + self.custom_hyperopt.populate_buy_trend) # type: ignore if hasattr(self.custom_hyperopt, 'populate_sell_trend'): - self.backtesting.strategy.advise_sell = \ - self.custom_hyperopt.populate_sell_trend # type: ignore + self.backtesting.strategy.advise_sell = ( # type: ignore + self.custom_hyperopt.populate_sell_trend) # type: ignore # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): @@ -162,6 +161,10 @@ class Hyperopt: self.num_epochs_saved = num_epochs logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " f"saved to '{self.results_file}'.") + # Store hyperopt filename + latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN) + file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)}, + log=False) @staticmethod def _read_results(results_file: Path) -> List: @@ -230,6 +233,9 @@ class Hyperopt: if space in ['buy', 'sell']: result_dict.setdefault('params', {}).update(space_params) elif space == 'roi': + # TODO: get rid of OrderedDict when support for python 3.6 will be + # dropped (dicts keep the order as the language feature) + # Convert keys in min_roi dict to strings because # rapidjson cannot dump dicts with integer keys... # OrderedDict is used to keep the numeric order of the items @@ -244,11 +250,29 @@ class Hyperopt: def _params_pretty_print(params, space: str, header: str) -> None: if space in params: space_params = Hyperopt._space_params(params, space, 5) + params_result = f"\n# {header}\n" if space == 'stoploss': - print(header, space_params.get('stoploss')) + params_result += f"stoploss = {space_params.get('stoploss')}" + elif space == 'roi': + # TODO: get rid of OrderedDict when support for python 3.6 will be + # dropped (dicts keep the order as the language feature) + minimal_roi_result = rapidjson.dumps( + OrderedDict( + (str(k), v) for k, v in space_params.items() + ), + default=str, indent=4, number_mode=rapidjson.NM_NATIVE) + params_result += f"minimal_roi = {minimal_roi_result}" + elif space == 'trailing': + + for k, v in space_params.items(): + params_result += f'{k} = {v}\n' + else: - print(header) - pprint(space_params, indent=4) + params_result += f"{space}_params = {pformat(space_params, indent=4)}" + params_result = params_result.replace("}", "\n}").replace("{", "{\n ") + + params_result = params_result.replace("\n", "\n ") + print(params_result) @staticmethod def _space_params(params, space: str, r: int = None) -> Dict: @@ -296,12 +320,18 @@ class Hyperopt: trials = json_normalize(results, max_level=1) trials['Best'] = '' + if 'results_metrics.winsdrawslosses' not in trials.columns: + # Ensure compatibility with older versions of hyperopt results + trials['results_metrics.winsdrawslosses'] = 'N/A' + trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', + 'results_metrics.winsdrawslosses', 'results_metrics.avg_profit', 'results_metrics.total_profit', 'results_metrics.profit', 'results_metrics.duration', 'loss', 'is_initial_point', 'is_best']] - trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', - 'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best'] + trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', + 'Total profit', 'Profit', 'Avg duration', 'Objective', + 'is_initial_point', 'is_best'] trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '* ' trials.loc[trials['is_best'], 'Best'] = 'Best' @@ -374,7 +404,7 @@ class Hyperopt: return # Verification for overwrite - if path.isfile(csv_file): + if Path(csv_file).is_file(): logger.error(f"CSV file already exists: {csv_file}") return @@ -478,16 +508,16 @@ class Hyperopt: params_details = self._get_params_details(params_dict) if self.has_space('roi'): - self.backtesting.strategy.minimal_roi = \ - self.custom_hyperopt.generate_roi_table(params_dict) + self.backtesting.strategy.minimal_roi = ( # type: ignore + self.custom_hyperopt.generate_roi_table(params_dict)) if self.has_space('buy'): - self.backtesting.strategy.advise_buy = \ - self.custom_hyperopt.buy_strategy_generator(params_dict) + self.backtesting.strategy.advise_buy = ( # type: ignore + self.custom_hyperopt.buy_strategy_generator(params_dict)) if self.has_space('sell'): - self.backtesting.strategy.advise_sell = \ - self.custom_hyperopt.sell_strategy_generator(params_dict) + self.backtesting.strategy.advise_sell = ( # type: ignore + self.custom_hyperopt.sell_strategy_generator(params_dict)) if self.has_space('stoploss'): self.backtesting.strategy.stoploss = params_dict['stoploss'] @@ -508,10 +538,12 @@ class Hyperopt: backtesting_results = self.backtesting.backtest( processed=processed, stake_amount=self.config['stake_amount'], - start_date=min_date, - end_date=max_date, + start_date=min_date.datetime, + end_date=max_date.datetime, max_open_trades=self.max_open_trades, position_stacking=self.position_stacking, + enable_protections=self.config.get('enable_protections', False), + ) return self._get_results_dict(backtesting_results, min_date, max_date, params_dict, params_details) @@ -542,9 +574,17 @@ class Hyperopt: } def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict: + wins = len(backtesting_results[backtesting_results.profit_percent > 0]) + draws = len(backtesting_results[backtesting_results.profit_percent == 0]) + losses = len(backtesting_results[backtesting_results.profit_percent < 0]) return { 'trade_count': len(backtesting_results.index), + 'wins': wins, + 'draws': draws, + 'losses': losses, + 'winsdrawslosses': f"{wins:>4} {draws:>4} {losses:>4}", 'avg_profit': backtesting_results.profit_percent.mean() * 100.0, + 'median_profit': backtesting_results.profit_percent.median() * 100.0, 'total_profit': backtesting_results.profit_abs.sum(), 'profit': backtesting_results.profit_percent.sum() * 100.0, 'duration': backtesting_results.trade_duration.mean(), @@ -556,7 +596,10 @@ class Hyperopt: """ stake_cur = self.config['stake_currency'] return (f"{results_metrics['trade_count']:6d} trades. " + f"{results_metrics['wins']}/{results_metrics['draws']}" + f"/{results_metrics['losses']} Wins/Draws/Losses. " f"Avg profit {results_metrics['avg_profit']: 6.2f}%. " + f"Median profit {results_metrics['median_profit']: 6.2f}%. " f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} " f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). " f"Avg duration {results_metrics['duration']:5.1f} min." @@ -609,17 +652,17 @@ class Hyperopt: preprocessed[pair] = trim_dataframe(df, timerange) min_date, max_date = get_timerange(data) - logger.info( - 'Hyperopting with data from %s up to %s (%s days)..', - min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days - ) + logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' + f'({(max_date - min_date).days} days)..') + dump(preprocessed, self.data_pickle_file) # We don't need exchange instance anymore while running hyperopt self.backtesting.exchange = None # type: ignore self.backtesting.pairlists = None # type: ignore - - self.epochs = self.load_previous_results(self.results_file) + self.backtesting.strategy.dp = None # type: ignore + IStrategy.dp = None # type: ignore cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") @@ -628,6 +671,10 @@ class Hyperopt: self.dimensions: List[Dimension] = self.hyperopt_space() self.opt = self.get_optimizer(self.dimensions, config_jobs) + + if self.print_colorized: + colorama_init(autoreset=True) + try: with Parallel(n_jobs=config_jobs) as parallel: jobs = parallel._effective_n_jobs() diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index b3cedef2c..b8c44ed59 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -13,6 +13,7 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import round_dict + logger = logging.getLogger(__name__) @@ -31,13 +32,15 @@ class IHyperOpt(ABC): Class attributes you can use: ticker_interval -> int: value of the ticker interval to use for the strategy """ - ticker_interval: str + ticker_interval: str # DEPRECATED + timeframe: str def __init__(self, config: dict) -> None: self.config = config # Assign ticker_interval to be used in hyperopt - IHyperOpt.ticker_interval = str(config['ticker_interval']) + IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED + IHyperOpt.timeframe = str(config['timeframe']) @staticmethod def buy_strategy_generator(params: Dict[str, Any]) -> Callable: @@ -218,9 +221,10 @@ class IHyperOpt(ABC): # Why do I still need such shamanic mantras in modern python? def __getstate__(self): state = self.__dict__.copy() - state['ticker_interval'] = self.ticker_interval + state['timeframe'] = self.timeframe return state def __setstate__(self, state): self.__dict__.update(state) - IHyperOpt.ticker_interval = state['ticker_interval'] + IHyperOpt.ticker_interval = state['timeframe'] + IHyperOpt.timeframe = state['timeframe'] diff --git a/freqtrade/optimize/hyperopt_loss_interface.py b/freqtrade/optimize/hyperopt_loss_interface.py index 879a9f0e9..48407a8a8 100644 --- a/freqtrade/optimize/hyperopt_loss_interface.py +++ b/freqtrade/optimize/hyperopt_loss_interface.py @@ -14,7 +14,7 @@ class IHyperOptLoss(ABC): Interface for freqtrade hyperopt Loss functions. Defines the custom loss function (`hyperopt_loss_function()` which is evaluated every epoch.) """ - ticker_interval: str + timeframe: str @staticmethod @abstractmethod diff --git a/freqtrade/optimize/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss_sharpe.py index 29377bdd5..232fb33b6 100644 --- a/freqtrade/optimize/hyperopt_loss_sharpe.py +++ b/freqtrade/optimize/hyperopt_loss_sharpe.py @@ -6,8 +6,8 @@ Hyperoptimization. """ from datetime import datetime -from pandas import DataFrame import numpy as np +from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/hyperopt_loss_sharpe_daily.py b/freqtrade/optimize/hyperopt_loss_sharpe_daily.py index e4cd1d749..bcba73a7f 100644 --- a/freqtrade/optimize/hyperopt_loss_sharpe_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sharpe_daily.py @@ -43,7 +43,7 @@ class SharpeHyperOptLossDaily(IHyperOptLoss): normalize=True) sum_daily = ( - results.resample(resample_freq, on='close_time').agg( + results.resample(resample_freq, on='close_date').agg( {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0) ) diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss_sortino.py index d470a9977..c0ff0773a 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino.py +++ b/freqtrade/optimize/hyperopt_loss_sortino.py @@ -6,8 +6,8 @@ Hyperoptimization. """ from datetime import datetime -from pandas import DataFrame import numpy as np +from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py index cd6a8bcc2..3b099a253 100644 --- a/freqtrade/optimize/hyperopt_loss_sortino_daily.py +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -45,7 +45,7 @@ class SortinoHyperOptLossDaily(IHyperOptLoss): normalize=True) sum_daily = ( - results.resample(resample_freq, on='close_time').agg( + results.resample(resample_freq, on='close_date').agg( {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0) ) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index d89860a73..d029ecd13 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -1,46 +1,41 @@ import logging -from datetime import timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Union +from arrow import Arrow +from numpy import int64 from pandas import DataFrame from tabulate import tabulate +from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN +from freqtrade.data.btanalysis import calculate_market_change, calculate_max_drawdown from freqtrade.misc import file_dump_json + logger = logging.getLogger(__name__) -def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame]) -> None: +def store_backtest_stats(recordfilename: Path, stats: Dict[str, DataFrame]) -> None: """ - Stores backtest results to file (one file per strategy) - :param recordfilename: Destination filename - :param all_results: Dict of Dataframes, one results dataframe per strategy + Stores backtest results + :param recordfilename: Path object, which can either be a filename or a directory. + Filenames will be appended with a timestamp right before the suffix + while for diectories, /backtest-result-.json will be used as filename + :param stats: Dataframe containing the backtesting statistics """ - for strategy, results in all_results.items(): - records = backtest_result_to_list(results) + if recordfilename.is_dir(): + filename = (recordfilename / + f'backtest-result-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}.json') + else: + filename = Path.joinpath( + recordfilename.parent, + f'{recordfilename.stem}-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' + ).with_suffix(recordfilename.suffix) + file_dump_json(filename, stats) - if records: - filename = recordfilename - if len(all_results) > 1: - # Inject strategy to filename - filename = Path.joinpath( - recordfilename.parent, - f'{recordfilename.stem}-{strategy}').with_suffix(recordfilename.suffix) - logger.info(f'Dumping backtest results to {filename}') - file_dump_json(filename, records) - - -def backtest_result_to_list(results: DataFrame) -> List[List]: - """ - Converts a list of Backtest-results to list - :param results: Dataframe containing results for one strategy - :return: List of Lists containing the trades - """ - return [[t.pair, t.profit_percent, t.open_time.timestamp(), - t.close_time.timestamp(), t.open_index - 1, t.trade_duration, - t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value] - for index, t in results.iterrows()] + latest_filename = Path.joinpath(filename.parent, LAST_BT_RESULT_FN) + file_dump_json(latest_filename, {'latest_backtest': str(filename.name)}) def _get_line_floatfmt() -> List[str]: @@ -63,15 +58,19 @@ def _generate_result_line(result: DataFrame, max_open_trades: int, first_column: """ Generate one result dict, with "first_column" as key. """ + profit_sum = result['profit_percent'].sum() + profit_total = profit_sum / max_open_trades + return { 'key': first_column, 'trades': len(result), - 'profit_mean': result['profit_percent'].mean(), - 'profit_mean_pct': result['profit_percent'].mean() * 100.0, - 'profit_sum': result['profit_percent'].sum(), - 'profit_sum_pct': result['profit_percent'].sum() * 100.0, + 'profit_mean': result['profit_percent'].mean() if len(result) > 0 else 0.0, + 'profit_mean_pct': result['profit_percent'].mean() * 100.0 if len(result) > 0 else 0.0, + 'profit_sum': profit_sum, + 'profit_sum_pct': round(profit_sum * 100.0, 2), 'profit_total_abs': result['profit_abs'].sum(), - 'profit_total_pct': result['profit_percent'].sum() * 100.0 / max_open_trades, + 'profit_total': profit_total, + 'profit_total_pct': round(profit_total * 100.0, 2), 'duration_avg': str(timedelta( minutes=round(result['trade_duration'].mean())) ) if not result.empty else '0:00', @@ -126,8 +125,8 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List result = results.loc[results['sell_reason'] == reason] profit_mean = result['profit_percent'].mean() - profit_sum = result["profit_percent"].sum() - profit_percent_tot = round(result['profit_percent'].sum() * 100.0 / max_open_trades, 2) + profit_sum = result['profit_percent'].sum() + profit_total = profit_sum / max_open_trades tabular_data.append( { @@ -141,25 +140,25 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List 'profit_sum': profit_sum, 'profit_sum_pct': round(profit_sum * 100, 2), 'profit_total_abs': result['profit_abs'].sum(), - 'profit_pct_total': profit_percent_tot, + 'profit_total': profit_total, + 'profit_total_pct': round(profit_total * 100, 2), } ) return tabular_data -def generate_strategy_metrics(stake_currency: str, max_open_trades: int, - all_results: Dict) -> List[Dict]: +def generate_strategy_metrics(all_results: Dict) -> List[Dict]: """ Generate summary per strategy - :param stake_currency: stake-currency - used to correctly name headers - :param max_open_trades: Maximum allowed open trades used for backtest :param all_results: Dict of containing results for all strategies :return: List of Dicts containing the metrics per Strategy """ tabular_data = [] for strategy, results in all_results.items(): - tabular_data.append(_generate_result_line(results, max_open_trades, strategy)) + tabular_data.append(_generate_result_line( + results['results'], results['config']['max_open_trades'], strategy) + ) return tabular_data @@ -189,19 +188,63 @@ def generate_edge_table(results: dict) -> str: floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore -def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame], - all_results: Dict[str, DataFrame]) -> Dict[str, Any]: +def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: + if len(results) == 0: + return { + 'backtest_best_day': 0, + 'backtest_worst_day': 0, + 'winning_days': 0, + 'draw_days': 0, + 'losing_days': 0, + 'winner_holding_avg': timedelta(), + 'loser_holding_avg': timedelta(), + } + daily_profit = results.resample('1d', on='close_date')['profit_percent'].sum() + worst = min(daily_profit) + best = max(daily_profit) + winning_days = sum(daily_profit > 0) + draw_days = sum(daily_profit == 0) + losing_days = sum(daily_profit < 0) + + winning_trades = results.loc[results['profit_percent'] > 0] + losing_trades = results.loc[results['profit_percent'] < 0] + + return { + 'backtest_best_day': best, + 'backtest_worst_day': worst, + 'winning_days': winning_days, + 'draw_days': draw_days, + 'losing_days': losing_days, + 'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean())) + if not winning_trades.empty else timedelta()), + 'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean())) + if not losing_trades.empty else timedelta()), + } + + +def generate_backtest_stats(btdata: Dict[str, DataFrame], + all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]], + min_date: Arrow, max_date: Arrow + ) -> Dict[str, Any]: """ - :param config: Configuration object used for backtest :param btdata: Backtest data - :param all_results: backtest result - dictionary with { Strategy: results}. + :param all_results: backtest result - dictionary in the form: + { Strategy: {'results: results, 'config: config}}. + :param min_date: Backtest start date + :param max_date: Backtest end date :return: Dictionary containing results per strategy and a stratgy summary. """ - stake_currency = config['stake_currency'] - max_open_trades = config['max_open_trades'] result: Dict[str, Any] = {'strategy': {}} - for strategy, results in all_results.items(): + market_change = calculate_market_change(btdata, 'close') + + for strategy, content in all_results.items(): + results: Dict[str, DataFrame] = content['results'] + if not isinstance(results, DataFrame): + continue + config = content['config'] + max_open_trades = config['max_open_trades'] + stake_currency = config['stake_currency'] pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency, max_open_trades=max_open_trades, @@ -212,17 +255,75 @@ def generate_backtest_stats(config: Dict, btdata: Dict[str, DataFrame], max_open_trades=max_open_trades, results=results.loc[results['open_at_end']], skip_nan=True) + daily_stats = generate_daily_stats(results) + best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'], + key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None + worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], + key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None + results['open_timestamp'] = results['open_date'].astype(int64) // 1e6 + results['close_timestamp'] = results['close_date'].astype(int64) // 1e6 + + backtest_days = (max_date - min_date).days strat_stats = { - 'trades': backtest_result_to_list(results), + 'trades': results.to_dict(orient='records'), + 'locks': [lock.to_json() for lock in content['locks']], + 'best_pair': best_pair, + 'worst_pair': worst_pair, 'results_per_pair': pair_results, 'sell_reason_summary': sell_reason_stats, 'left_open_trades': left_open_results, - } + 'total_trades': len(results), + 'profit_mean': results['profit_percent'].mean() if len(results) > 0 else 0, + 'profit_total': results['profit_percent'].sum(), + 'profit_total_abs': results['profit_abs'].sum(), + 'backtest_start': min_date.datetime, + 'backtest_start_ts': min_date.int_timestamp * 1000, + 'backtest_end': max_date.datetime, + 'backtest_end_ts': max_date.int_timestamp * 1000, + 'backtest_days': backtest_days, + + 'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0, + 'market_change': market_change, + 'pairlist': list(btdata.keys()), + 'stake_amount': config['stake_amount'], + 'stake_currency': config['stake_currency'], + 'max_open_trades': (config['max_open_trades'] + if config['max_open_trades'] != float('inf') else -1), + 'timeframe': config['timeframe'], + # Parameters relevant for backtesting + 'stoploss': config['stoploss'], + 'trailing_stop': config.get('trailing_stop', False), + 'trailing_stop_positive': config.get('trailing_stop_positive'), + 'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0), + 'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False), + 'minimal_roi': config['minimal_roi'], + 'use_sell_signal': config['ask_strategy']['use_sell_signal'], + 'sell_profit_only': config['ask_strategy']['sell_profit_only'], + 'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'], + **daily_stats, + } result['strategy'][strategy] = strat_stats - strategy_results = generate_strategy_metrics(stake_currency=stake_currency, - max_open_trades=max_open_trades, - all_results=all_results) + try: + max_drawdown, drawdown_start, drawdown_end = calculate_max_drawdown( + results, value_col='profit_percent') + strat_stats.update({ + 'max_drawdown': max_drawdown, + 'drawdown_start': drawdown_start, + 'drawdown_start_ts': drawdown_start.timestamp() * 1000, + 'drawdown_end': drawdown_end, + 'drawdown_end_ts': drawdown_end.timestamp() * 1000, + }) + except ValueError: + strat_stats.update({ + 'max_drawdown': 0.0, + 'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc), + 'drawdown_start_ts': 0, + 'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc), + 'drawdown_end_ts': 0, + }) + + strategy_results = generate_strategy_metrics(all_results=all_results) result['strategy_comparison'] = strategy_results @@ -273,7 +374,7 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren output = [[ t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'], - t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_pct_total'], + t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'], t['profit_total_pct'], ] for t in sell_reason_stats] return tabulate(output, headers=headers, tablefmt="orgtbl", stralign="right") @@ -298,6 +399,45 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str: floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") +def text_table_add_metrics(strat_results: Dict) -> str: + if len(strat_results['trades']) > 0: + best_trade = max(strat_results['trades'], key=lambda x: x['profit_percent']) + worst_trade = min(strat_results['trades'], key=lambda x: x['profit_percent']) + metrics = [ + ('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)), + ('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)), + ('Max open trades', strat_results['max_open_trades']), + ('', ''), # Empty line to improve readability + ('Total trades', strat_results['total_trades']), + ('Total Profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"), + ('Trades per day', strat_results['trades_per_day']), + ('', ''), # Empty line to improve readability + ('Best Pair', f"{strat_results['best_pair']['key']} " + f"{round(strat_results['best_pair']['profit_sum_pct'], 2)}%"), + ('Worst Pair', f"{strat_results['worst_pair']['key']} " + f"{round(strat_results['worst_pair']['profit_sum_pct'], 2)}%"), + ('Best trade', f"{best_trade['pair']} {round(best_trade['profit_percent'] * 100, 2)}%"), + ('Worst trade', f"{worst_trade['pair']} " + f"{round(worst_trade['profit_percent'] * 100, 2)}%"), + + ('Best day', f"{round(strat_results['backtest_best_day'] * 100, 2)}%"), + ('Worst day', f"{round(strat_results['backtest_worst_day'] * 100, 2)}%"), + ('Days win/draw/lose', f"{strat_results['winning_days']} / " + f"{strat_results['draw_days']} / {strat_results['losing_days']}"), + ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), + ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"), + ('', ''), # Empty line to improve readability + ('Max Drawdown', f"{round(strat_results['max_drawdown'] * 100, 2)}%"), + ('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)), + ('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)), + ('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"), + ] + + return tabulate(metrics, headers=["Metric", "Value"], tablefmt="orgtbl") + else: + return '' + + def show_backtest_results(config: Dict, backtest_stats: Dict): stake_currency = config['stake_currency'] @@ -312,15 +452,21 @@ def show_backtest_results(config: Dict, backtest_stats: Dict): table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'], stake_currency=stake_currency) - if isinstance(table, str): + if isinstance(table, str) and len(table) > 0: print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '=')) print(table) table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency) - if isinstance(table, str): + if isinstance(table, str) and len(table) > 0: print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) print(table) - if isinstance(table, str): + + table = text_table_add_metrics(results) + if isinstance(table, str) and len(table) > 0: + print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '=')) + print(table) + + if isinstance(table, str) and len(table) > 0: print('=' * len(table.splitlines()[0])) print() diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py deleted file mode 100644 index b85d68269..000000000 --- a/freqtrade/pairlist/PriceFilter.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Price pair list filter -""" -import logging -from typing import Any, Dict - -from freqtrade.pairlist.IPairList import IPairList - - -logger = logging.getLogger(__name__) - - -class PriceFilter(IPairList): - - def __init__(self, exchange, pairlistmanager, - config: Dict[str, Any], pairlistconfig: Dict[str, Any], - pairlist_pos: int) -> None: - super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) - - self._low_price_ratio = pairlistconfig.get('low_price_ratio', 0) - self._enabled = self._low_price_ratio != 0 - - @property - def needstickers(self) -> bool: - """ - Boolean property defining if tickers are necessary. - If no Pairlist requries tickers, an empty List is passed - as tickers argument to filter_pairlist - """ - return True - - def short_desc(self) -> str: - """ - Short whitelist method description - used for startup-messages - """ - return f"{self.name} - Filtering pairs priced below {self._low_price_ratio * 100}%." - - def _validate_pair(self, ticker) -> bool: - """ - Check if if one price-step (pip) is > than a certain barrier. - :param ticker: ticker dict as returned from ccxt.load_markets() - :return: True if the pair can stay, false if it should be removed - """ - if ticker['last'] is None: - self.log_on_refresh(logger.info, - f"Removed {ticker['symbol']} from whitelist, because " - "ticker['last'] is empty (Usually no trade in the last 24h).") - return False - compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last']) - changeperc = compare / ticker['last'] - if changeperc > self._low_price_ratio: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because 1 unit is {changeperc * 100:.3f}%") - return False - return True diff --git a/freqtrade/persistence/__init__.py b/freqtrade/persistence/__init__.py new file mode 100644 index 000000000..35f2bc406 --- /dev/null +++ b/freqtrade/persistence/__init__.py @@ -0,0 +1,4 @@ +# flake8: noqa: F401 + +from freqtrade.persistence.models import Order, Trade, clean_dry_run_db, cleanup_db, init_db +from freqtrade.persistence.pairlock_middleware import PairLocks diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py new file mode 100644 index 000000000..ed976c2a9 --- /dev/null +++ b/freqtrade/persistence/migrations.py @@ -0,0 +1,150 @@ +import logging +from typing import List + +from sqlalchemy import inspect + + +logger = logging.getLogger(__name__) + + +def get_table_names_for_table(inspector, tabletype): + return [t for t in inspector.get_table_names() if t.startswith(tabletype)] + + +def has_column(columns: List, searchname: str) -> bool: + return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 + + +def get_column_def(columns: List, column: str, default: str) -> str: + return default if not has_column(columns, column) else column + + +def get_backup_name(tabs, backup_prefix: str): + table_back_name = backup_prefix + for i, table_back_name in enumerate(tabs): + table_back_name = f'{backup_prefix}{i}' + logger.debug(f'trying {table_back_name}') + + return table_back_name + + +def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, cols: List): + fee_open = get_column_def(cols, 'fee_open', 'fee') + fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') + fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') + fee_close = get_column_def(cols, 'fee_close', 'fee') + fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') + fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') + open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') + close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') + stop_loss = get_column_def(cols, 'stop_loss', '0.0') + stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') + initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') + initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') + stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') + stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') + max_rate = get_column_def(cols, 'max_rate', '0.0') + min_rate = get_column_def(cols, 'min_rate', 'null') + sell_reason = get_column_def(cols, 'sell_reason', 'null') + strategy = get_column_def(cols, 'strategy', 'null') + # If ticker-interval existed use that, else null. + if has_column(cols, 'ticker_interval'): + timeframe = get_column_def(cols, 'timeframe', 'ticker_interval') + else: + timeframe = get_column_def(cols, 'timeframe', 'null') + + open_trade_value = get_column_def(cols, 'open_trade_value', + f'amount * open_rate * (1 + {fee_open})') + close_profit_abs = get_column_def( + cols, 'close_profit_abs', + f"(amount * close_rate * (1 - {fee_close})) - {open_trade_value}") + sell_order_status = get_column_def(cols, 'sell_order_status', 'null') + amount_requested = get_column_def(cols, 'amount_requested', 'amount') + + # Schema migration necessary + engine.execute(f"alter table trades rename to {table_back_name}") + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + engine.execute(f"drop index {index['name']}") + # let SQLAlchemy create the schema as required + decl_base.metadata.create_all(engine) + + # Copy data back - following the correct schema + engine.execute(f"""insert into trades + (id, exchange, pair, is_open, + fee_open, fee_open_cost, fee_open_currency, + fee_close, fee_close_cost, fee_open_currency, open_rate, + open_rate_requested, close_rate, close_rate_requested, close_profit, + stake_amount, amount, amount_requested, open_date, close_date, open_order_id, + stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, + stoploss_order_id, stoploss_last_update, + max_rate, min_rate, sell_reason, sell_order_status, strategy, + timeframe, open_trade_value, close_profit_abs + ) + select id, lower(exchange), + case + when instr(pair, '_') != 0 then + substr(pair, instr(pair, '_') + 1) || '/' || + substr(pair, 1, instr(pair, '_') - 1) + else pair + end + pair, + is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, + {fee_open_currency} fee_open_currency, {fee_close} fee_close, + {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, + open_rate, {open_rate_requested} open_rate_requested, close_rate, + {close_rate_requested} close_rate_requested, close_profit, + stake_amount, amount, {amount_requested}, open_date, close_date, open_order_id, + {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, + {initial_stop_loss} initial_stop_loss, + {initial_stop_loss_pct} initial_stop_loss_pct, + {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, + {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, + {sell_order_status} sell_order_status, + {strategy} strategy, {timeframe} timeframe, + {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs + from {table_back_name} + """) + + +def migrate_open_orders_to_trades(engine): + engine.execute(""" + insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open) + select id ft_trade_id, pair ft_pair, open_order_id, + case when close_rate_requested is null then 'buy' + else 'sell' end ft_order_side, 1 ft_is_open + from trades + where open_order_id is not null + union all + select id ft_trade_id, pair ft_pair, stoploss_order_id order_id, + 'stoploss' ft_order_side, 1 ft_is_open + from trades + where stoploss_order_id is not null + """) + + +def check_migrate(engine, decl_base, previous_tables) -> None: + """ + Checks if migration is necessary and migrates if necessary + """ + inspector = inspect(engine) + + cols = inspector.get_columns('trades') + tabs = get_table_names_for_table(inspector, 'trades') + table_back_name = get_backup_name(tabs, 'trades_bak') + + # Check for latest column + if not has_column(cols, 'open_trade_value'): + logger.info(f'Running database migration for trades - backup: {table_back_name}') + migrate_trades_table(decl_base, inspector, engine, table_back_name, cols) + # Reread columns - the above recreated the table! + inspector = inspect(engine) + cols = inspector.get_columns('trades') + + if 'orders' not in previous_tables: + logger.info('Moving open orders to Orders table.') + migrate_open_orders_to_trades(engine) + else: + pass + # Empty for now - as there is only one iteration of the orders table so far. + # table_back_name = get_backup_name(tabs, 'orders_bak') diff --git a/freqtrade/persistence.py b/freqtrade/persistence/models.py similarity index 55% rename from freqtrade/persistence.py rename to freqtrade/persistence/models.py index 823bf6dc0..7fa894e9c 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence/models.py @@ -2,21 +2,26 @@ This module contains the class to persist trades into SQLite """ import logging -from datetime import datetime +from datetime import datetime, timezone from decimal import Decimal from typing import Any, Dict, List, Optional import arrow -from sqlalchemy import (Boolean, Column, DateTime, Float, Integer, String, +from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Query +from sqlalchemy.orm import Query, relationship from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool +from sqlalchemy.sql.schema import UniqueConstraint + +from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.misc import safe_value_fallback +from freqtrade.persistence.migrations import check_migrate -from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) @@ -25,7 +30,7 @@ _DECL_BASE: Any = declarative_base() _SQL_DOCS_URL = 'http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls' -def init(db_url: str, clean_open_orders: bool = False) -> None: +def init_db(db_url: str, clean_open_orders: bool = False) -> None: """ Initializes this module with the given config, registers all known command handlers @@ -56,116 +61,22 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: # We should use the scoped_session object - not a seperately initialized version Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) Trade.query = Trade.session.query_property() + # Copy session attributes to order object too + Order.session = Trade.session + Order.query = Order.session.query_property() + PairLock.session = Trade.session + PairLock.query = PairLock.session.query_property() + + previous_tables = inspect(engine).get_table_names() _DECL_BASE.metadata.create_all(engine) - check_migrate(engine) + check_migrate(engine, decl_base=_DECL_BASE, previous_tables=previous_tables) # Clean dry_run DB if the db is not in-memory if clean_open_orders and db_url != 'sqlite://': clean_dry_run_db() -def has_column(columns: List, searchname: str) -> bool: - return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 - - -def get_column_def(columns: List, column: str, default: str) -> str: - return default if not has_column(columns, column) else column - - -def check_migrate(engine) -> None: - """ - Checks if migration is necessary and migrates if necessary - """ - inspector = inspect(engine) - - cols = inspector.get_columns('trades') - tabs = inspector.get_table_names() - table_back_name = 'trades_bak' - for i, table_back_name in enumerate(tabs): - table_back_name = f'trades_bak{i}' - logger.debug(f'trying {table_back_name}') - - # Check for latest column - if not has_column(cols, 'sell_order_status'): - logger.info(f'Running database migration - backup available as {table_back_name}') - - fee_open = get_column_def(cols, 'fee_open', 'fee') - fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') - fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') - fee_close = get_column_def(cols, 'fee_close', 'fee') - fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') - fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') - open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') - close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') - stop_loss = get_column_def(cols, 'stop_loss', '0.0') - stop_loss_pct = get_column_def(cols, 'stop_loss_pct', 'null') - initial_stop_loss = get_column_def(cols, 'initial_stop_loss', '0.0') - initial_stop_loss_pct = get_column_def(cols, 'initial_stop_loss_pct', 'null') - stoploss_order_id = get_column_def(cols, 'stoploss_order_id', 'null') - stoploss_last_update = get_column_def(cols, 'stoploss_last_update', 'null') - max_rate = get_column_def(cols, 'max_rate', '0.0') - min_rate = get_column_def(cols, 'min_rate', 'null') - sell_reason = get_column_def(cols, 'sell_reason', 'null') - strategy = get_column_def(cols, 'strategy', 'null') - ticker_interval = get_column_def(cols, 'ticker_interval', 'null') - open_trade_price = get_column_def(cols, 'open_trade_price', - f'amount * open_rate * (1 + {fee_open})') - close_profit_abs = get_column_def( - cols, 'close_profit_abs', - f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") - sell_order_status = get_column_def(cols, 'sell_order_status', 'null') - - # Schema migration necessary - engine.execute(f"alter table trades rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") - # let SQLAlchemy create the schema as required - _DECL_BASE.metadata.create_all(engine) - - # Copy data back - following the correct schema - engine.execute(f"""insert into trades - (id, exchange, pair, is_open, - fee_open, fee_open_cost, fee_open_currency, - fee_close, fee_close_cost, fee_open_currency, open_rate, - open_rate_requested, close_rate, close_rate_requested, close_profit, - stake_amount, amount, open_date, close_date, open_order_id, - stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, - stoploss_order_id, stoploss_last_update, - max_rate, min_rate, sell_reason, sell_order_status, strategy, - ticker_interval, open_trade_price, close_profit_abs - ) - select id, lower(exchange), - case - when instr(pair, '_') != 0 then - substr(pair, instr(pair, '_') + 1) || '/' || - substr(pair, 1, instr(pair, '_') - 1) - else pair - end - pair, - is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, - {fee_open_currency} fee_open_currency, {fee_close} fee_close, - {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, - open_rate, {open_rate_requested} open_rate_requested, close_rate, - {close_rate_requested} close_rate_requested, close_profit, - stake_amount, amount, open_date, close_date, open_order_id, - {stop_loss} stop_loss, {stop_loss_pct} stop_loss_pct, - {initial_stop_loss} initial_stop_loss, - {initial_stop_loss_pct} initial_stop_loss_pct, - {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, - {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, - {sell_order_status} sell_order_status, - {strategy} strategy, {ticker_interval} ticker_interval, - {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs - from {table_back_name} - """) - - # Reread columns - the above recreated the table! - inspector = inspect(engine) - cols = inspector.get_columns('trades') - - -def cleanup() -> None: +def cleanup_db() -> None: """ Flushes all pending operations to disk. :return: None @@ -184,13 +95,121 @@ def clean_dry_run_db() -> None: trade.open_order_id = None +class Order(_DECL_BASE): + """ + Order database model + Keeps a record of all orders placed on the exchange + + One to many relationship with Trades: + - One trade can have many orders + - One Order can only be associated with one Trade + + Mirrors CCXT Order structure + """ + __tablename__ = 'orders' + # Uniqueness should be ensured over pair, order_id + # its likely that order_id is unique per Pair on some exchanges. + __table_args__ = (UniqueConstraint('ft_pair', 'order_id', name="_order_pair_order_id"),) + + id = Column(Integer, primary_key=True) + ft_trade_id = Column(Integer, ForeignKey('trades.id'), index=True) + + trade = relationship("Trade", back_populates="orders") + + ft_order_side = Column(String, nullable=False) + ft_pair = Column(String, nullable=False) + ft_is_open = Column(Boolean, nullable=False, default=True, index=True) + + order_id = Column(String, nullable=False, index=True) + status = Column(String, nullable=True) + symbol = Column(String, nullable=True) + order_type = Column(String, nullable=True) + side = Column(String, nullable=True) + price = Column(Float, nullable=True) + amount = Column(Float, nullable=True) + filled = Column(Float, nullable=True) + remaining = Column(Float, nullable=True) + cost = Column(Float, nullable=True) + order_date = Column(DateTime, nullable=True, default=datetime.utcnow) + order_filled_date = Column(DateTime, nullable=True) + order_update_date = Column(DateTime, nullable=True) + + def __repr__(self): + + return (f'Order(id={self.id}, order_id={self.order_id}, trade_id={self.ft_trade_id}, ' + f'side={self.side}, order_type={self.order_type}, status={self.status})') + + def update_from_ccxt_object(self, order): + """ + Update Order from ccxt response + Only updates if fields are available from ccxt - + """ + if self.order_id != str(order['id']): + raise DependencyException("Order-id's don't match") + + self.status = order.get('status', self.status) + self.symbol = order.get('symbol', self.symbol) + self.order_type = order.get('type', self.order_type) + self.side = order.get('side', self.side) + self.price = order.get('price', self.price) + self.amount = order.get('amount', self.amount) + self.filled = order.get('filled', self.filled) + self.remaining = order.get('remaining', self.remaining) + self.cost = order.get('cost', self.cost) + if 'timestamp' in order and order['timestamp'] is not None: + self.order_date = datetime.fromtimestamp(order['timestamp'] / 1000, tz=timezone.utc) + + self.ft_is_open = True + if self.status in ('closed', 'canceled', 'cancelled'): + self.ft_is_open = False + if order.get('filled', 0) > 0: + self.order_filled_date = arrow.utcnow().datetime + self.order_update_date = arrow.utcnow().datetime + + @staticmethod + def update_orders(orders: List['Order'], order: Dict[str, Any]): + """ + Get all non-closed orders - useful when trying to batch-update orders + """ + filtered_orders = [o for o in orders if o.order_id == order.get('id')] + if filtered_orders: + oobj = filtered_orders[0] + oobj.update_from_ccxt_object(order) + else: + logger.warning(f"Did not find order for {order}.") + + @staticmethod + def parse_from_ccxt_object(order: Dict[str, Any], pair: str, side: str) -> 'Order': + """ + Parse an order from a ccxt object and return a new order Object. + """ + o = Order(order_id=str(order['id']), ft_order_side=side, ft_pair=pair) + + o.update_from_ccxt_object(order) + return o + + @staticmethod + def get_open_orders() -> List['Order']: + """ + """ + return Order.query.filter(Order.ft_is_open.is_(True)).all() + + class Trade(_DECL_BASE): """ - Class used to define a trade structure + Trade database model. + Also handles updating and querying trades """ __tablename__ = 'trades' + use_db: bool = True + # Trades container for backtesting + trades: List['Trade'] = [] + id = Column(Integer, primary_key=True) + + orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan") + exchange = Column(String, nullable=False) pair = Column(String, nullable=False, index=True) is_open = Column(Boolean, nullable=False, default=True, index=True) @@ -202,14 +221,15 @@ class Trade(_DECL_BASE): fee_close_currency = Column(String, nullable=True) open_rate = Column(Float) open_rate_requested = Column(Float) - # open_trade_price - calculated via _calc_open_trade_price - open_trade_price = Column(Float) + # open_trade_value - calculated via _calc_open_trade_value + open_trade_value = Column(Float) close_rate = Column(Float) close_rate_requested = Column(Float) close_profit = Column(Float) close_profit_abs = Column(Float) stake_amount = Column(Float, nullable=False) amount = Column(Float) + amount_requested = Column(Float) open_date = Column(DateTime, nullable=False, default=datetime.utcnow) close_date = Column(DateTime) open_order_id = Column(String) @@ -232,14 +252,14 @@ class Trade(_DECL_BASE): sell_reason = Column(String, nullable=True) sell_order_status = Column(String, nullable=True) strategy = Column(String, nullable=True) - ticker_interval = Column(Integer, nullable=True) + timeframe = Column(Integer, nullable=True) def __init__(self, **kwargs): super().__init__(**kwargs) - self.recalc_open_trade_price() + self.recalc_open_trade_value() def __repr__(self): - open_since = self.open_date.strftime('%Y-%m-%d %H:%M:%S') if self.is_open else 'closed' + open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' return (f'Trade(id={self.id}, pair={self.pair}, amount={self.amount:.8f}, ' f'open_rate={self.open_rate:.8f}, open_since={open_since})') @@ -251,9 +271,10 @@ class Trade(_DECL_BASE): 'is_open': self.is_open, 'exchange': self.exchange, 'amount': round(self.amount, 8), + 'amount_requested': round(self.amount_requested, 8) if self.amount_requested else None, 'stake_amount': round(self.stake_amount, 8), 'strategy': self.strategy, - 'ticker_interval': self.ticker_interval, + 'timeframe': self.timeframe, 'fee_open': self.fee_open, 'fee_open_cost': self.fee_open_cost, @@ -263,34 +284,38 @@ class Trade(_DECL_BASE): 'fee_close_currency': self.fee_close_currency, 'open_date_hum': arrow.get(self.open_date).humanize(), - 'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"), - 'open_timestamp': int(self.open_date.timestamp() * 1000), + 'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT), + 'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000), 'open_rate': self.open_rate, 'open_rate_requested': self.open_rate_requested, - 'open_trade_price': self.open_trade_price, + 'open_trade_value': round(self.open_trade_value, 8), 'close_date_hum': (arrow.get(self.close_date).humanize() if self.close_date else None), - 'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S") + 'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT) if self.close_date else None), - 'close_timestamp': int(self.close_date.timestamp() * 1000) if self.close_date else None, + 'close_timestamp': int(self.close_date.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.close_date else None, 'close_rate': self.close_rate, 'close_rate_requested': self.close_rate_requested, - 'close_profit': self.close_profit, - 'close_profit_abs': self.close_profit_abs, + 'close_profit': self.close_profit, # Deprecated + 'close_profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None, + 'close_profit_abs': self.close_profit_abs, # Deprecated + + 'profit_ratio': self.close_profit, + 'profit_pct': round(self.close_profit * 100, 2) if self.close_profit else None, + 'profit_abs': self.close_profit_abs, 'sell_reason': self.sell_reason, 'sell_order_status': self.sell_order_status, - 'stop_loss': self.stop_loss, # Deprecated - should not be used 'stop_loss_abs': self.stop_loss, 'stop_loss_ratio': self.stop_loss_pct if self.stop_loss_pct else None, 'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None, 'stoploss_order_id': self.stoploss_order_id, - 'stoploss_last_update': (self.stoploss_last_update.strftime("%Y-%m-%d %H:%M:%S") + 'stoploss_last_update': (self.stoploss_last_update.strftime(DATETIME_PRINT_FORMAT) if self.stoploss_last_update else None), - 'stoploss_last_update_timestamp': (int(self.stoploss_last_update.timestamp() * 1000) - if self.stoploss_last_update else None), - 'initial_stop_loss': self.initial_stop_loss, # Deprecated - should not be used + 'stoploss_last_update_timestamp': int(self.stoploss_last_update.replace( + tzinfo=timezone.utc).timestamp() * 1000) if self.stoploss_last_update else None, 'initial_stop_loss_abs': self.initial_stop_loss, 'initial_stop_loss_ratio': (self.initial_stop_loss_pct if self.initial_stop_loss_pct else None), @@ -302,6 +327,14 @@ class Trade(_DECL_BASE): 'open_order_id': self.open_order_id, } + @staticmethod + def reset_trades() -> None: + """ + Resets all trades. Only active for backtesting mode. + """ + if not Trade.use_db: + Trade.trades = [] + def adjust_min_max_rates(self, current_price: float) -> None: """ Adjust the max_rate and min_rate. @@ -354,36 +387,39 @@ class Trade(_DECL_BASE): def update(self, order: Dict) -> None: """ Updates this entity with amount and actual open/close rates. - :param order: order retrieved by exchange.get_order() + :param order: order retrieved by exchange.fetch_order() :return: None """ order_type = order['type'] # Ignore open and cancelled orders - if order['status'] == 'open' or order['price'] is None: + if order['status'] == 'open' or safe_value_fallback(order, 'average', 'price') is None: return logger.info('Updating trade (id=%s) ...', self.id) if order_type in ('market', 'limit') and order['side'] == 'buy': # Update open rate and actual amount - self.open_rate = Decimal(order['price']) - self.amount = Decimal(order.get('filled', order['amount'])) - self.recalc_open_trade_price() - logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self) + self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price')) + self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount')) + self.recalc_open_trade_value() + if self.is_open: + logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.') self.open_order_id = None elif order_type in ('market', 'limit') and order['side'] == 'sell': - self.close(order['price']) - logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self) - elif order_type in ('stop_loss_limit', 'stop-loss'): + if self.is_open: + logger.info(f'{order_type.upper()}_SELL has been fulfilled for {self}.') + self.close(safe_value_fallback(order, 'average', 'price')) + elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss - logger.info('%s is hit for %s.', order_type.upper(), self) + if self.is_open: + logger.info(f'{order_type.upper()} is hit for {self}.') self.close(order['average']) else: raise ValueError(f'Unknown order type: {order_type}') - cleanup() + cleanup_db() - def close(self, rate: float) -> None: + def close(self, rate: float, *, show_msg: bool = True) -> None: """ Sets close_rate to the given rate, calculates total profit and marks trade as closed @@ -391,14 +427,15 @@ class Trade(_DECL_BASE): self.close_rate = Decimal(rate) self.close_profit = self.calc_profit_ratio() self.close_profit_abs = self.calc_profit() - self.close_date = datetime.utcnow() + self.close_date = self.close_date or datetime.utcnow() self.is_open = False self.sell_order_status = 'closed' self.open_order_id = None - logger.info( - 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', - self - ) + if show_msg: + logger.info( + 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', + self + ) def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], side: str) -> None: @@ -429,7 +466,18 @@ class Trade(_DECL_BASE): else: return False - def _calc_open_trade_price(self) -> float: + def update_order(self, order: Dict) -> None: + Order.update_orders(self.orders, order) + + def delete(self) -> None: + + for order in self.orders: + Order.session.delete(order) + + Trade.session.delete(self) + Trade.session.flush() + + def _calc_open_trade_value(self) -> float: """ Calculate the open_rate including open_fee. :return: Price in of the open trade incl. Fees @@ -438,14 +486,14 @@ class Trade(_DECL_BASE): fees = buy_trade * Decimal(self.fee_open) return float(buy_trade + fees) - def recalc_open_trade_price(self) -> None: + def recalc_open_trade_value(self) -> None: """ - Recalculate open_trade_price. + Recalculate open_trade_value. Must be called whenever open_rate or fee_open is changed. """ - self.open_trade_price = self._calc_open_trade_price() + self.open_trade_value = self._calc_open_trade_value() - def calc_close_trade_price(self, rate: Optional[float] = None, + def calc_close_trade_value(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: """ Calculate the close_rate including fee @@ -472,11 +520,11 @@ class Trade(_DECL_BASE): If rate is not set self.close_rate will be used :return: profit in stake currency as float """ - close_trade_price = self.calc_close_trade_price( + close_trade_value = self.calc_close_trade_value( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - profit = close_trade_price - self.open_trade_price + profit = close_trade_value - self.open_trade_value return float(f"{profit:.8f}") def calc_profit_ratio(self, rate: Optional[float] = None, @@ -488,13 +536,28 @@ class Trade(_DECL_BASE): :param fee: fee to use on the close rate (optional). :return: profit ratio as float """ - close_trade_price = self.calc_close_trade_price( + close_trade_value = self.calc_close_trade_value( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - profit_ratio = (close_trade_price / self.open_trade_price) - 1 + profit_ratio = (close_trade_value / self.open_trade_value) - 1 return float(f"{profit_ratio:.8f}") + def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]: + """ + Finds latest order for this orderside and status + :param order_side: Side of the order (either 'buy' or 'sell') + :param is_open: Only search for open orders? + :return: latest Order object if it exists, else None + """ + orders = [o for o in self.orders if o.side == order_side] + if is_open is not None: + orders = [o for o in orders if o.ft_is_open == is_open] + if len(orders) > 0: + return orders[-1] + else: + return None + @staticmethod def get_trades(trade_filter=None) -> Query: """ @@ -512,6 +575,43 @@ class Trade(_DECL_BASE): else: return Trade.query + @staticmethod + def get_trades_proxy(*, pair: str = None, is_open: bool = None, + open_date: datetime = None, close_date: datetime = None, + ) -> List['Trade']: + """ + Helper function to query Trades. + Returns a List of trades, filtered on the parameters given. + In live mode, converts the filter to a database query and returns all rows + In Backtest mode, uses filters on Trade.trades to get the result. + + :return: unsorted List[Trade] + """ + if Trade.use_db: + trade_filter = [] + if pair: + trade_filter.append(Trade.pair == pair) + if open_date: + trade_filter.append(Trade.open_date > open_date) + if close_date: + trade_filter.append(Trade.close_date > close_date) + if is_open is not None: + trade_filter.append(Trade.is_open.is_(is_open)) + return Trade.get_trades(trade_filter).all() + else: + # Offline mode - without database + sel_trades = [trade for trade in Trade.trades] + if pair: + sel_trades = [trade for trade in sel_trades if trade.pair == pair] + if open_date: + sel_trades = [trade for trade in sel_trades if trade.open_date > open_date] + if close_date: + sel_trades = [trade for trade in sel_trades if trade.close_date + and trade.close_date > close_date] + if is_open is not None: + sel_trades = [trade for trade in sel_trades if trade.is_open == is_open] + return sel_trades + @staticmethod def get_open_trades() -> List[Any]: """ @@ -526,6 +626,26 @@ class Trade(_DECL_BASE): """ return Trade.get_trades(Trade.open_order_id.isnot(None)).all() + @staticmethod + def get_open_trades_without_assigned_fees(): + """ + Returns all open trades which don't have open fees set correctly + """ + return Trade.get_trades([Trade.fee_open_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(True), + ]).all() + + @staticmethod + def get_sold_trades_without_assigned_fees(): + """ + Returns all closed trades which don't have fees set correctly + """ + return Trade.get_trades([Trade.fee_close_currency.is_(None), + Trade.orders.any(), + Trade.is_open.is_(False), + ]).all() + @staticmethod def total_open_trades_stakes() -> float: """ @@ -590,3 +710,56 @@ class Trade(_DECL_BASE): trade.stop_loss = None trade.adjust_stop_loss(trade.open_rate, desired_stoploss) logger.info(f"New stoploss: {trade.stop_loss}.") + + +class PairLock(_DECL_BASE): + """ + Pair Locks database model. + """ + __tablename__ = 'pairlocks' + + id = Column(Integer, primary_key=True) + + pair = Column(String, nullable=False, index=True) + reason = Column(String, nullable=True) + # Time the pair was locked (start time) + lock_time = Column(DateTime, nullable=False) + # Time until the pair is locked (end time) + lock_end_time = Column(DateTime, nullable=False, index=True) + + active = Column(Boolean, nullable=False, default=True, index=True) + + def __repr__(self): + lock_time = self.lock_time.strftime(DATETIME_PRINT_FORMAT) + lock_end_time = self.lock_end_time.strftime(DATETIME_PRINT_FORMAT) + return (f'PairLock(id={self.id}, pair={self.pair}, lock_time={lock_time}, ' + f'lock_end_time={lock_end_time})') + + @staticmethod + def query_pair_locks(pair: Optional[str], now: datetime) -> Query: + """ + Get all currently active locks for this pair + :param pair: Pair to check for. Returns all current locks if pair is empty + :param now: Datetime object (generated via datetime.now(timezone.utc)). + """ + + filters = [PairLock.lock_end_time > now, + # Only active locks + PairLock.active.is_(True), ] + if pair: + filters.append(PairLock.pair == pair) + return PairLock.query.filter( + *filters + ) + + def to_json(self) -> Dict[str, Any]: + return { + 'pair': self.pair, + 'lock_time': self.lock_time.strftime(DATETIME_PRINT_FORMAT), + 'lock_timestamp': int(self.lock_time.replace(tzinfo=timezone.utc).timestamp() * 1000), + 'lock_end_time': self.lock_end_time.strftime(DATETIME_PRINT_FORMAT), + 'lock_end_timestamp': int(self.lock_end_time.replace(tzinfo=timezone.utc + ).timestamp() * 1000), + 'reason': self.reason, + 'active': self.active, + } diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py new file mode 100644 index 000000000..8644146d8 --- /dev/null +++ b/freqtrade/persistence/pairlock_middleware.py @@ -0,0 +1,125 @@ +import logging +from datetime import datetime, timezone +from typing import List, Optional + +from freqtrade.exchange import timeframe_to_next_date +from freqtrade.persistence.models import PairLock + + +logger = logging.getLogger(__name__) + + +class PairLocks(): + """ + Pairlocks middleware class + Abstracts the database layer away so it becomes optional - which will be necessary to support + backtesting and hyperopt in the future. + """ + + use_db = True + locks: List[PairLock] = [] + + timeframe: str = '' + + @staticmethod + def reset_locks() -> None: + """ + Resets all locks. Only active for backtesting mode. + """ + if not PairLocks.use_db: + PairLocks.locks = [] + + @staticmethod + def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None: + """ + Create PairLock from now to "until". + Uses database by default, unless PairLocks.use_db is set to False, + in which case a list is maintained. + :param pair: pair to lock. use '*' to lock all pairs + :param until: End time of the lock. Will be rounded up to the next candle. + :param reason: Reason string that will be shown as reason for the lock + :param now: Current timestamp. Used to determine lock start time. + """ + lock = PairLock( + pair=pair, + lock_time=now or datetime.now(timezone.utc), + lock_end_time=timeframe_to_next_date(PairLocks.timeframe, until), + reason=reason, + active=True + ) + if PairLocks.use_db: + PairLock.session.add(lock) + PairLock.session.flush() + else: + PairLocks.locks.append(lock) + + @staticmethod + def get_pair_locks(pair: Optional[str], now: Optional[datetime] = None) -> List[PairLock]: + """ + Get all currently active locks for this pair + :param pair: Pair to check for. Returns all current locks if pair is empty + :param now: Datetime object (generated via datetime.now(timezone.utc)). + defaults to datetime.now(timezone.utc) + """ + if not now: + now = datetime.now(timezone.utc) + + if PairLocks.use_db: + return PairLock.query_pair_locks(pair, now).all() + else: + locks = [lock for lock in PairLocks.locks if ( + lock.lock_end_time >= now + and lock.active is True + and (pair is None or lock.pair == pair) + )] + return locks + + @staticmethod + def get_pair_longest_lock(pair: str, now: Optional[datetime] = None) -> Optional[PairLock]: + """ + Get the lock that expires the latest for the pair given. + """ + locks = PairLocks.get_pair_locks(pair, now) + locks = sorted(locks, key=lambda l: l.lock_end_time, reverse=True) + return locks[0] if locks else None + + @staticmethod + def unlock_pair(pair: str, now: Optional[datetime] = None) -> None: + """ + Release all locks for this pair. + :param pair: Pair to unlock + :param now: Datetime object (generated via datetime.now(timezone.utc)). + defaults to datetime.now(timezone.utc) + """ + if not now: + now = datetime.now(timezone.utc) + + logger.info(f"Releasing all locks for {pair}.") + locks = PairLocks.get_pair_locks(pair, now) + for lock in locks: + lock.active = False + if PairLocks.use_db: + PairLock.session.flush() + + @staticmethod + def is_global_lock(now: Optional[datetime] = None) -> bool: + """ + :param now: Datetime object (generated via datetime.now(timezone.utc)). + defaults to datetime.now(timezone.utc) + """ + if not now: + now = datetime.now(timezone.utc) + + return len(PairLocks.get_pair_locks('*', now)) > 0 + + @staticmethod + def is_pair_locked(pair: str, now: Optional[datetime] = None) -> bool: + """ + :param pair: Pair to check for + :param now: Datetime object (generated via datetime.now(timezone.utc)). + defaults to datetime.now(timezone.utc) + """ + if not now: + now = datetime.now(timezone.utc) + + return len(PairLocks.get_pair_locks(pair, now)) > 0 or PairLocks.is_global_lock(now) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index d519c5f4e..497218deb 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -5,69 +5,81 @@ from typing import Any, Dict, List import pandas as pd from freqtrade.configuration import TimeRange -from freqtrade.data.btanalysis import (calculate_max_drawdown, - combine_dataframes_with_mean, - create_cum_profit, - extract_trades_of_period, load_trades) +from freqtrade.data.btanalysis import (calculate_max_drawdown, combine_dataframes_with_mean, + create_cum_profit, extract_trades_of_period, load_trades) from freqtrade.data.converter import trim_dataframe -from freqtrade.data.history import load_data +from freqtrade.data.dataprovider import DataProvider +from freqtrade.data.history import get_timerange, load_data from freqtrade.exceptions import OperationalException -from freqtrade.exchange import timeframe_to_prev_date +from freqtrade.exchange import timeframe_to_prev_date, timeframe_to_seconds from freqtrade.misc import pair_to_filename -from freqtrade.resolvers import StrategyResolver +from freqtrade.resolvers import ExchangeResolver, StrategyResolver +from freqtrade.strategy import IStrategy + logger = logging.getLogger(__name__) try: - from plotly.subplots import make_subplots - from plotly.offline import plot import plotly.graph_objects as go + from plotly.offline import plot + from plotly.subplots import make_subplots except ImportError: logger.exception("Module plotly not found \n Please install using `pip3 install plotly`") exit(1) -def init_plotscript(config): +def init_plotscript(config, startup_candles: int = 0): """ Initialize objects needed for plotting :return: Dict with candle (OHLCV) data, trades and pairs """ if "pairs" in config: - pairs = config["pairs"] + pairs = config['pairs'] else: - pairs = config["exchange"]["pair_whitelist"] + pairs = config['exchange']['pair_whitelist'] # Set timerange to use - timerange = TimeRange.parse_timerange(config.get("timerange")) + timerange = TimeRange.parse_timerange(config.get('timerange')) data = load_data( - datadir=config.get("datadir"), + datadir=config.get('datadir'), pairs=pairs, - timeframe=config.get('ticker_interval', '5m'), + timeframe=config.get('timeframe', '5m'), timerange=timerange, + startup_candles=startup_candles, data_format=config.get('dataformat_ohlcv', 'json'), ) + if startup_candles: + min_date, max_date = get_timerange(data) + logger.info(f"Loading data from {min_date} to {max_date}") + timerange.adjust_start_if_necessary(timeframe_to_seconds(config.get('timeframe', '5m')), + startup_candles, min_date) + no_trades = False + filename = config.get('exportfilename') if config.get('no_trades', False): no_trades = True - elif not config['exportfilename'].is_file() and config['trade_source'] == 'file': - logger.warning("Backtest file is missing skipping trades.") - no_trades = True + elif config['trade_source'] == 'file': + if not filename.is_dir() and not filename.is_file(): + logger.warning("Backtest file is missing skipping trades.") + no_trades = True trades = load_trades( config['trade_source'], db_url=config.get('db_url'), - exportfilename=config.get('exportfilename'), - no_trades=no_trades + exportfilename=filename, + no_trades=no_trades, + strategy=config.get('strategy'), ) - trades = trim_dataframe(trades, timerange, 'open_time') + trades = trim_dataframe(trades, timerange, 'open_date') return {"ohlcv": data, "trades": trades, "pairs": pairs, + "timerange": timerange, } @@ -163,10 +175,11 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: if trades is not None and len(trades) > 0: # Create description for sell summarizing the trade trades['desc'] = trades.apply(lambda row: f"{round(row['profit_percent'] * 100, 1)}%, " - f"{row['sell_reason']}, {row['duration']} min", + f"{row['sell_reason']}, " + f"{row['trade_duration']} min", axis=1) trade_buys = go.Scatter( - x=trades["open_time"], + x=trades["open_date"], y=trades["open_rate"], mode='markers', name='Trade buy', @@ -181,7 +194,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: ) trade_sells = go.Scatter( - x=trades.loc[trades['profit_percent'] > 0, "close_time"], + x=trades.loc[trades['profit_percent'] > 0, "close_date"], y=trades.loc[trades['profit_percent'] > 0, "close_rate"], text=trades.loc[trades['profit_percent'] > 0, "desc"], mode='markers', @@ -194,7 +207,7 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: ) ) trade_sells_loss = go.Scatter( - x=trades.loc[trades['profit_percent'] <= 0, "close_time"], + x=trades.loc[trades['profit_percent'] <= 0, "close_date"], y=trades.loc[trades['profit_percent'] <= 0, "close_rate"], text=trades.loc[trades['profit_percent'] <= 0, "desc"], mode='markers', @@ -250,6 +263,65 @@ def create_plotconfig(indicators1: List[str], indicators2: List[str], return plot_config +def plot_area(fig, row: int, data: pd.DataFrame, indicator_a: str, + indicator_b: str, label: str = "", + fill_color: str = "rgba(0,176,246,0.2)") -> make_subplots: + """ Creates a plot for the area between two traces and adds it to fig. + :param fig: Plot figure to append to + :param row: row number for this plot + :param data: candlestick DataFrame + :param indicator_a: indicator name as populated in stragetie + :param indicator_b: indicator name as populated in stragetie + :param label: label for the filled area + :param fill_color: color to be used for the filled area + :return: fig with added filled_traces plot + """ + if indicator_a in data and indicator_b in data: + # make lines invisible to get the area plotted, only. + line = {'color': 'rgba(255,255,255,0)'} + # TODO: Figure out why scattergl causes problems plotly/plotly.js#2284 + trace_a = go.Scatter(x=data.date, y=data[indicator_a], + showlegend=False, + line=line) + trace_b = go.Scatter(x=data.date, y=data[indicator_b], name=label, + fill="tonexty", fillcolor=fill_color, + line=line) + fig.add_trace(trace_a, row, 1) + fig.add_trace(trace_b, row, 1) + return fig + + +def add_areas(fig, row: int, data: pd.DataFrame, indicators) -> make_subplots: + """ Adds all area plots (specified in plot_config) to fig. + :param fig: Plot figure to append to + :param row: row number for this plot + :param data: candlestick DataFrame + :param indicators: dict with indicators. ie.: plot_config['main_plot'] or + plot_config['subplots'][subplot_label] + :return: fig with added filled_traces plot + """ + for indicator, ind_conf in indicators.items(): + if 'fill_to' in ind_conf: + indicator_b = ind_conf['fill_to'] + if indicator in data and indicator_b in data: + label = ind_conf.get('fill_label', + f'{indicator}<>{indicator_b}') + fill_color = ind_conf.get('fill_color', 'rgba(0,176,246,0.2)') + fig = plot_area(fig, row, data, indicator, indicator_b, + label=label, fill_color=fill_color) + elif indicator not in data: + logger.info( + 'Indicator "%s" ignored. Reason: This indicator is not ' + 'found in your strategy.', indicator + ) + elif indicator_b not in data: + logger.info( + 'fill_to: "%s" ignored. Reason: This indicator is not ' + 'in your strategy.', indicator_b + ) + return fig + + def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFrame = None, *, indicators1: List[str] = [], indicators2: List[str] = [], @@ -267,7 +339,6 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra :return: Plotly figure """ plot_config = create_plotconfig(indicators1, indicators2, plot_config) - rows = 2 + len(plot_config['subplots']) row_widths = [1 for _ in plot_config['subplots']] # Define the graph @@ -333,36 +404,20 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra fig.add_trace(sells, 1, 1) else: logger.warning("No sell-signals found.") - - # TODO: Figure out why scattergl causes problems plotly/plotly.js#2284 - if 'bb_lowerband' in data and 'bb_upperband' in data: - bb_lower = go.Scatter( - x=data.date, - y=data.bb_lowerband, - showlegend=False, - line={'color': 'rgba(255,255,255,0)'}, - ) - bb_upper = go.Scatter( - x=data.date, - y=data.bb_upperband, - name='Bollinger Band', - fill="tonexty", - fillcolor="rgba(0,176,246,0.2)", - line={'color': 'rgba(255,255,255,0)'}, - ) - fig.add_trace(bb_lower, 1, 1) - fig.add_trace(bb_upper, 1, 1) - if ('bb_upperband' in plot_config['main_plot'] - and 'bb_lowerband' in plot_config['main_plot']): - del plot_config['main_plot']['bb_upperband'] - del plot_config['main_plot']['bb_lowerband'] - - # Add indicators to main plot + # Add Bollinger Bands + fig = plot_area(fig, 1, data, 'bb_lowerband', 'bb_upperband', + label="Bollinger Band") + # prevent bb_lower and bb_upper from plotting + try: + del plot_config['main_plot']['bb_lowerband'] + del plot_config['main_plot']['bb_upperband'] + except KeyError: + pass + # main plot goes to row 1 fig = add_indicators(fig=fig, row=1, indicators=plot_config['main_plot'], data=data) - + fig = add_areas(fig, 1, data, plot_config['main_plot']) fig = plot_trades(fig, trades) - - # Volume goes to row 2 + # sub plot: Volume goes to row 2 volume = go.Bar( x=data['date'], y=data['volume'], @@ -371,13 +426,14 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra marker_line_color='DarkSlateGrey' ) fig.add_trace(volume, 2, 1) - - # Add indicators to separate row - for i, name in enumerate(plot_config['subplots']): - fig = add_indicators(fig=fig, row=3 + i, - indicators=plot_config['subplots'][name], + # add each sub plot to a separate row + for i, label in enumerate(plot_config['subplots']): + sub_config = plot_config['subplots'][label] + row = 3 + i + fig = add_indicators(fig=fig, row=row, indicators=sub_config, data=data) - + # fill area between indicators ( 'fill_to': 'other_indicator') + fig = add_areas(fig, row, data, sub_config) return fig @@ -467,7 +523,10 @@ def load_and_plot_trades(config: Dict[str, Any]): """ strategy = StrategyResolver.load_strategy(config) - plot_elements = init_plotscript(config) + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) + IStrategy.dp = DataProvider(config, exchange) + plot_elements = init_plotscript(config, strategy.startup_candle_count) + timerange = plot_elements['timerange'] trades = plot_elements['trades'] pair_counter = 0 for pair, data in plot_elements["ohlcv"].items(): @@ -475,6 +534,7 @@ def load_and_plot_trades(config: Dict[str, Any]): logger.info("analyse pair %s", pair) df_analyzed = strategy.analyze_ticker(data, {'pair': pair}) + df_analyzed = trim_dataframe(df_analyzed, timerange) trades_pair = trades.loc[trades['pair'] == pair] trades_pair = extract_trades_of_period(df_analyzed, trades_pair) @@ -482,13 +542,13 @@ def load_and_plot_trades(config: Dict[str, Any]): pair=pair, data=df_analyzed, trades=trades_pair, - indicators1=config.get("indicators1", []), - indicators2=config.get("indicators2", []), + indicators1=config.get('indicators1', []), + indicators2=config.get('indicators2', []), plot_config=strategy.plot_config if hasattr(strategy, 'plot_config') else {} ) - store_plot_file(fig, filename=generate_plot_filename(pair, config['ticker_interval']), - directory=config['user_data_dir'] / "plot") + store_plot_file(fig, filename=generate_plot_filename(pair, config['timeframe']), + directory=config['user_data_dir'] / 'plot') logger.info('End of plotting process. %s plots generated', pair_counter) @@ -505,8 +565,8 @@ def plot_profit(config: Dict[str, Any]) -> None: # Filter trades to relevant pairs # Remove open pairs - we don't know the profit yet so can't calculate profit for these. # Also, If only one open pair is left, then the profit-generation would fail. - trades = trades[(trades['pair'].isin(plot_elements["pairs"])) - & (~trades['close_time'].isnull()) + trades = trades[(trades['pair'].isin(plot_elements['pairs'])) + & (~trades['close_date'].isnull()) ] if len(trades) == 0: raise OperationalException("No trades found, cannot generate Profit-plot without " @@ -514,7 +574,7 @@ def plot_profit(config: Dict[str, Any]) -> None: # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend - fig = generate_profit_graph(plot_elements["pairs"], plot_elements["ohlcv"], - trades, config.get('ticker_interval', '5m')) + fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'], + trades, config.get('timeframe', '5m')) store_plot_file(fig, filename='freqtrade-profit-plot.html', - directory=config['user_data_dir'] / "plot", auto_open=True) + directory=config['user_data_dir'] / 'plot', auto_open=True) diff --git a/freqtrade/pairlist/__init__.py b/freqtrade/plugins/__init__.py similarity index 100% rename from freqtrade/pairlist/__init__.py rename to freqtrade/plugins/__init__.py diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py new file mode 100644 index 000000000..8c3a5d22f --- /dev/null +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -0,0 +1,99 @@ +""" +Minimum age (days listed) pair list filter +""" +import logging +from copy import deepcopy +from typing import Any, Dict, List, Optional + +import arrow +from pandas import DataFrame + +from freqtrade.exceptions import OperationalException +from freqtrade.misc import plural +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class AgeFilter(IPairList): + + # Checked symbols cache (dictionary of ticker symbol => timestamp) + _symbolsChecked: Dict[str, int] = {} + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._min_days_listed = pairlistconfig.get('min_days_listed', 10) + + if self._min_days_listed < 1: + raise OperationalException("AgeFilter requires min_days_listed to be >= 1") + if self._min_days_listed > exchange.ohlcv_candle_limit: + raise OperationalException("AgeFilter requires min_days_listed to not exceed " + "exchange max request size " + f"({exchange.ohlcv_candle_limit})") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Filtering pairs with age less than " + f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.") + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked] + if not needed_pairs: + return pairlist + + since_ms = int(arrow.utcnow() + .floor('day') + .shift(days=-self._min_days_listed - 1) + .float_timestamp) * 1000 + candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) + if self._enabled: + for p in deepcopy(pairlist): + daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None + if not self._validate_pair_loc(p, daily_candles): + pairlist.remove(p) + logger.info(f"Validated {len(pairlist)} pairs.") + return pairlist + + def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: + """ + Validate age for the ticker + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, false if it should be removed + """ + # Check symbol in cache + if pair in self._symbolsChecked: + return True + + if daily_candles is not None: + if len(daily_candles) > self._min_days_listed: + # We have fetched at least the minimum required number of daily candles + # Add to cache, store the time we last checked this symbol + self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000 + return True + else: + self.log_once(f"Removed {pair} from whitelist, because age " + f"{len(daily_candles)} is less than {self._min_days_listed} " + f"{plural(self._min_days_listed, 'day')}", logger.info) + return False + return False diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py similarity index 85% rename from freqtrade/pairlist/IPairList.py rename to freqtrade/plugins/pairlist/IPairList.py index f48a7dcfd..865aa90d6 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -6,16 +6,15 @@ from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy from typing import Any, Dict, List -from cachetools import TTLCache, cached - from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active +from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) -class IPairList(ABC): +class IPairList(LoggingMixin, ABC): def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], @@ -36,7 +35,7 @@ class IPairList(ABC): self._pairlist_pos = pairlist_pos self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) self._last_refresh = 0 - self._log_cache = TTLCache(maxsize=1024, ttl=self.refresh_period) + LoggingMixin.__init__(self, logger, self.refresh_period) @property def name(self) -> str: @@ -46,29 +45,11 @@ class IPairList(ABC): """ return self.__class__.__name__ - def log_on_refresh(self, logmethod, message: str) -> None: - """ - Logs message - not more often than "refresh_period" to avoid log spamming - Logs the log-message as debug as well to simplify debugging. - :param logmethod: Function that'll be called. Most likely `logger.info`. - :param message: String containing the message to be sent to the function. - :return: None. - """ - - @cached(cache=self._log_cache) - def _log_on_refresh(message: str): - logmethod(message) - - # Log as debug first - logger.debug(message) - # Call hidden function. - _log_on_refresh(message) - @abstractproperty def needstickers(self) -> bool: """ Boolean property defining if tickers are necessary. - If no Pairlist requries tickers, an empty List is passed + If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ @@ -79,13 +60,14 @@ class IPairList(ABC): -> Please overwrite in subclasses """ - def _validate_pair(self, ticker) -> bool: + def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: """ Check one pair against Pairlist Handler's specific conditions. Either implement it in the Pairlist Handler or override the generic filter_pairlist() method. + :param pair: Pair that's currently validated :param ticker: ticker dict as returned from ccxt.load_markets() :return: True if the pair can stay, false if it should be removed """ @@ -128,7 +110,7 @@ class IPairList(ABC): # Copy list since we're modifying this list for p in deepcopy(pairlist): # Filter out assets - if not self._validate_pair(tickers[p]): + if not self._validate_pair(p, tickers[p] if p in tickers else {}): pairlist.remove(p) return pairlist @@ -150,6 +132,9 @@ class IPairList(ABC): black_listed """ markets = self._exchange.markets + if not markets: + raise OperationalException( + 'Markets not loaded. Make sure that exchange is initialized correctly.') sanitized_whitelist: List[str] = [] for pair in pairlist: @@ -159,6 +144,11 @@ class IPairList(ABC): f"{self._exchange.name}. Removing it from whitelist..") continue + if not self._exchange.market_is_tradable(markets[pair]): + logger.warning(f"Pair {pair} is not tradable with Freqtrade." + "Removing it from whitelist..") + continue + if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']: logger.warning(f"Pair {pair} is not compatible with your stake currency " f"{self._config['stake_currency']}. Removing it from whitelist..") diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py new file mode 100644 index 000000000..7d91bb77c --- /dev/null +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -0,0 +1,66 @@ +""" +Performance pair list filter +""" +import logging +from typing import Any, Dict, List + +import pandas as pd + +from freqtrade.persistence import Trade +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class PerformanceFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short allowlist method description - used for startup-messages + """ + return f"{self.name} - Sorting pairs by performance." + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the allowlist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + # Get the trading performance for pairs from database + performance = pd.DataFrame(Trade.get_overall_performance()) + + # Skip performance-based sorting if no performance data is available + if len(performance) == 0: + return pairlist + + # Get pairlist from performance dataframe values + list_df = pd.DataFrame({'pair': pairlist}) + + # Set initial value for pairs with no trades to 0 + # Sort the list using: + # - primarily performance (high to low) + # - then count (low to high, so as to favor same performance with fewer trades) + # - then pair name alphametically + sorted_df = list_df.merge(performance, on='pair', how='left')\ + .fillna(0).sort_values(by=['count', 'pair'], ascending=True)\ + .sort_values(by=['profit'], ascending=False) + pairlist = sorted_df['pair'].tolist() + + return pairlist diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py similarity index 61% rename from freqtrade/pairlist/PrecisionFilter.py rename to freqtrade/plugins/pairlist/PrecisionFilter.py index 0331347be..519337f29 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -4,7 +4,8 @@ Precision pair list filter import logging from typing import Any, Dict -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -17,6 +18,10 @@ class PrecisionFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + if 'stoploss' not in self._config: + raise OperationalException( + 'PrecisionFilter can only work with stoploss defined. Please add the ' + 'stoploss key to your configuration (overwrites eventual strategy settings).') self._stoploss = self._config['stoploss'] self._enabled = self._stoploss != 0 @@ -27,7 +32,7 @@ class PrecisionFilter(IPairList): def needstickers(self) -> bool: """ Boolean property defining if tickers are necessary. - If no Pairlist requries tickers, an empty List is passed + If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ return True @@ -38,25 +43,25 @@ class PrecisionFilter(IPairList): """ return f"{self.name} - Filtering untradable pairs." - def _validate_pair(self, ticker: dict) -> bool: + def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: """ Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very low value pairs. + :param pair: Pair that's currently validated :param ticker: ticker dict as returned from ccxt.load_markets() - :return: True if the pair can stay, False if it should be removed + :return: True if the pair can stay, false if it should be removed """ stop_price = ticker['ask'] * self._stoploss # Adjust stop-prices to precision - sp = self._exchange.price_to_precision(ticker["symbol"], stop_price) + sp = self._exchange.price_to_precision(pair, stop_price) - stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99) + stop_gap_price = self._exchange.price_to_precision(pair, stop_price * 0.99) logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") if sp <= stop_gap_price: - self.log_on_refresh(logger.info, - f"Removed {ticker['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {stop_gap_price}") + self.log_once(f"Removed {ticker['symbol']} from whitelist, because " + f"stop price {sp} would be <= stop limit {stop_gap_price}", logger.info) return False return True diff --git a/freqtrade/plugins/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py new file mode 100644 index 000000000..6558f196f --- /dev/null +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -0,0 +1,96 @@ +""" +Price pair list filter +""" +import logging +from typing import Any, Dict + +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class PriceFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._low_price_ratio = pairlistconfig.get('low_price_ratio', 0) + if self._low_price_ratio < 0: + raise OperationalException("PriceFilter requires low_price_ratio to be >= 0") + self._min_price = pairlistconfig.get('min_price', 0) + if self._min_price < 0: + raise OperationalException("PriceFilter requires min_price to be >= 0") + self._max_price = pairlistconfig.get('max_price', 0) + if self._max_price < 0: + raise OperationalException("PriceFilter requires max_price to be >= 0") + self._enabled = ((self._low_price_ratio > 0) or + (self._min_price > 0) or + (self._max_price > 0)) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return True + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + active_price_filters = [] + if self._low_price_ratio != 0: + active_price_filters.append(f"below {self._low_price_ratio * 100}%") + if self._min_price != 0: + active_price_filters.append(f"below {self._min_price:.8f}") + if self._max_price != 0: + active_price_filters.append(f"above {self._max_price:.8f}") + + if len(active_price_filters): + return f"{self.name} - Filtering pairs priced {' or '.join(active_price_filters)}." + + return f"{self.name} - No price filters configured." + + def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: + """ + Check if if one price-step (pip) is > than a certain barrier. + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, false if it should be removed + """ + if ticker['last'] is None or ticker['last'] == 0: + self.log_once(f"Removed {pair} from whitelist, because " + "ticker['last'] is empty (Usually no trade in the last 24h).", + logger.info) + return False + + # Perform low_price_ratio check. + if self._low_price_ratio != 0: + compare = self._exchange.price_get_one_pip(pair, ticker['last']) + changeperc = compare / ticker['last'] + if changeperc > self._low_price_ratio: + self.log_once(f"Removed {pair} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%", logger.info) + return False + + # Perform min_price check. + if self._min_price != 0: + if ticker['last'] < self._min_price: + self.log_once(f"Removed {pair} from whitelist, " + f"because last price < {self._min_price:.8f}", logger.info) + return False + + # Perform max_price check. + if self._max_price != 0: + if ticker['last'] > self._max_price: + self.log_once(f"Removed {ticker['symbol']} from whitelist, " + f"because last price > {self._max_price:.8f}", logger.info) + return False + + return True diff --git a/freqtrade/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py similarity index 92% rename from freqtrade/pairlist/ShuffleFilter.py rename to freqtrade/plugins/pairlist/ShuffleFilter.py index ba3792213..4d3dd29e3 100644 --- a/freqtrade/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -5,7 +5,7 @@ import logging import random from typing import Any, Dict, List -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -25,7 +25,7 @@ class ShuffleFilter(IPairList): def needstickers(self) -> bool: """ Boolean property defining if tickers are necessary. - If no Pairlist requries tickers, an empty List is passed + If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ return False diff --git a/freqtrade/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py similarity index 72% rename from freqtrade/pairlist/SpreadFilter.py rename to freqtrade/plugins/pairlist/SpreadFilter.py index 0147c0068..2f3fe47e3 100644 --- a/freqtrade/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -4,7 +4,7 @@ Spread pair list filter import logging from typing import Any, Dict -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ class SpreadFilter(IPairList): def needstickers(self) -> bool: """ Boolean property defining if tickers are necessary. - If no Pairlist requries tickers, an empty List is passed + If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ return True @@ -36,18 +36,19 @@ class SpreadFilter(IPairList): return (f"{self.name} - Filtering pairs with ask/bid diff above " f"{self._max_spread_ratio * 100}%.") - def _validate_pair(self, ticker: dict) -> bool: + def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: """ Validate spread for the ticker + :param pair: Pair that's currently validated :param ticker: ticker dict as returned from ccxt.load_markets() - :return: True if the pair can stay, False if it should be removed + :return: True if the pair can stay, false if it should be removed """ if 'bid' in ticker and 'ask' in ticker: spread = 1 - ticker['bid'] / ticker['ask'] if spread > self._max_spread_ratio: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because spread {spread * 100:.3f}% >" - f"{self._max_spread_ratio * 100}%") + self.log_once(f"Removed {pair} from whitelist, because spread " + f"{spread * 100:.3f}% > {self._max_spread_ratio * 100}%", + logger.info) return False else: return True diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py similarity index 82% rename from freqtrade/pairlist/StaticPairList.py rename to freqtrade/plugins/pairlist/StaticPairList.py index b5c1bc767..dd592e0ca 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -7,7 +7,7 @@ import logging from typing import Any, Dict, List from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -24,11 +24,13 @@ class StaticPairList(IPairList): raise OperationalException(f"{self.name} can only be used in the first position " "in the list of Pairlist Handlers.") + self._allow_inactive = self._pairlistconfig.get('allow_inactive', False) + @property def needstickers(self) -> bool: """ Boolean property defining if tickers are necessary. - If no Pairlist requries tickers, an empty List is passed + If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ return False @@ -47,7 +49,10 @@ class StaticPairList(IPairList): :param tickers: Tickers (from exchange.get_tickers()). :return: List of pairs """ - return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) + if self._allow_inactive: + return self._config['exchange']['pair_whitelist'] + else: + return self._whitelist_for_active_markets(self._config['exchange']['pair_whitelist']) def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py similarity index 90% rename from freqtrade/pairlist/VolumePairList.py rename to freqtrade/plugins/pairlist/VolumePairList.py index d32be3dc9..dd8fc64fd 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -8,13 +8,13 @@ from datetime import datetime from typing import Any, Dict, List from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) -SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] +SORT_VALUES = ['quoteVolume'] class VolumePairList(IPairList): @@ -45,16 +45,11 @@ class VolumePairList(IPairList): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') - if self._sort_key != 'quoteVolume': - logger.warning( - "DEPRECATED: using any key other than quoteVolume for VolumePairList is deprecated." - ) - @property def needstickers(self) -> bool: """ Boolean property defining if tickers are necessary. - If no Pairlist requries tickers, an empty List is passed + If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ return True @@ -116,6 +111,6 @@ class VolumePairList(IPairList): # Limit pairlist to the requested number of pairs pairs = pairs[:self._number_pairs] - self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") + self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info) return pairs diff --git a/tests/pairlist/__init__.py b/freqtrade/plugins/pairlist/__init__.py similarity index 100% rename from tests/pairlist/__init__.py rename to freqtrade/plugins/pairlist/__init__.py diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py new file mode 100644 index 000000000..f2e84930b --- /dev/null +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -0,0 +1,108 @@ +""" +Rate of change pairlist filter +""" +import logging +from copy import deepcopy +from typing import Any, Dict, List, Optional + +import arrow +from cachetools.ttl import TTLCache +from pandas import DataFrame + +from freqtrade.exceptions import OperationalException +from freqtrade.misc import plural +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class RangeStabilityFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._days = pairlistconfig.get('lookback_days', 10) + self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01) + self._refresh_period = pairlistconfig.get('refresh_period', 1440) + + self._pair_cache: TTLCache = TTLCache(maxsize=100, ttl=self._refresh_period) + + if self._days < 1: + raise OperationalException("RangeStabilityFilter requires lookback_days to be >= 1") + if self._days > exchange.ohlcv_candle_limit: + raise OperationalException("RangeStabilityFilter requires lookback_days to not " + "exceed exchange max request size " + f"({exchange.ohlcv_candle_limit})") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Filtering pairs with rate of change below " + f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.") + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Validate trading range + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] + + since_ms = int(arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .float_timestamp) * 1000 + # Get all candles + candles = {} + if needed_pairs: + candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, + cache=False) + + if self._enabled: + for p in deepcopy(pairlist): + daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None + if not self._validate_pair_loc(p, daily_candles): + pairlist.remove(p) + return pairlist + + def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: + """ + Validate trading range + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, false if it should be removed + """ + # Check symbol in cache + if pair in self._pair_cache: + return self._pair_cache[pair] + + result = False + if daily_candles is not None and not daily_candles.empty: + highest_high = daily_candles['high'].max() + lowest_low = daily_candles['low'].min() + pct_change = ((highest_high - lowest_low) / lowest_low) if lowest_low > 0 else 0 + if pct_change >= self._min_rate_of_change: + result = True + else: + self.log_once(f"Removed {pair} from whitelist, because rate of change " + f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " + f"which is below the threshold of {self._min_rate_of_change}.", + logger.info) + result = False + self._pair_cache[pair] = result + + return result diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py similarity index 90% rename from freqtrade/pairlist/pairlistmanager.py rename to freqtrade/plugins/pairlistmanager.py index f532f2cd0..b71f02898 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -3,14 +3,14 @@ PairList manager class """ import logging from copy import deepcopy -from typing import Dict, List +from typing import Any, Dict, List from cachetools import TTLCache, cached -from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.IPairList import IPairList -from freqtrade.resolvers import PairListResolver from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.IPairList import IPairList +from freqtrade.resolvers import PairListResolver logger = logging.getLogger(__name__) @@ -26,9 +26,6 @@ class PairListManager(): self._pairlist_handlers: List[IPairList] = [] self._tickers_needed = False for pairlist_handler_config in self._config.get('pairlists', None): - if 'method' not in pairlist_handler_config: - logger.warning(f"No method found in {pairlist_handler_config}, ignoring.") - continue pairlist_handler = PairListResolver.load_pairlist( pairlist_handler_config['method'], exchange=exchange, @@ -100,7 +97,7 @@ class PairListManager(): self._whitelist = pairlist - def _prepare_whitelist(self, pairlist: List[str], tickers) -> List[str]: + def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]: """ Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove pairs that do not have ticker available @@ -131,6 +128,6 @@ class PairListManager(): def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: """ - Create list of pair tuples with (pair, ticker_interval) + Create list of pair tuples with (pair, timeframe) """ - return [(pair, timeframe or self._config['ticker_interval']) for pair in pairs] + return [(pair, timeframe or self._config['timeframe']) for pair in pairs] diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py new file mode 100644 index 000000000..a8edd4e4b --- /dev/null +++ b/freqtrade/plugins/protectionmanager.py @@ -0,0 +1,72 @@ +""" +Protection manager class +""" +import logging +from datetime import datetime, timezone +from typing import Dict, List, Optional + +from freqtrade.persistence import PairLocks +from freqtrade.plugins.protections import IProtection +from freqtrade.resolvers import ProtectionResolver + + +logger = logging.getLogger(__name__) + + +class ProtectionManager(): + + def __init__(self, config: dict) -> None: + self._config = config + + self._protection_handlers: List[IProtection] = [] + for protection_handler_config in self._config.get('protections', []): + protection_handler = ProtectionResolver.load_protection( + protection_handler_config['method'], + config=config, + protection_config=protection_handler_config, + ) + self._protection_handlers.append(protection_handler) + + if not self._protection_handlers: + logger.info("No protection Handlers defined.") + + @property + def name_list(self) -> List[str]: + """ + Get list of loaded Protection Handler names + """ + return [p.name for p in self._protection_handlers] + + def short_desc(self) -> List[Dict]: + """ + List of short_desc for each Pairlist Handler + """ + return [{p.name: p.short_desc()} for p in self._protection_handlers] + + def global_stop(self, now: Optional[datetime] = None) -> bool: + if not now: + now = datetime.now(timezone.utc) + result = False + for protection_handler in self._protection_handlers: + if protection_handler.has_global_stop: + result, until, reason = protection_handler.global_stop(now) + + # Early stopping - first positive result blocks further trades + if result and until: + if not PairLocks.is_global_lock(until): + PairLocks.lock_pair('*', until, reason, now=now) + result = True + return result + + def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool: + if not now: + now = datetime.now(timezone.utc) + result = False + for protection_handler in self._protection_handlers: + if protection_handler.has_local_stop: + result, until, reason = protection_handler.stop_per_pair(pair, now) + if result and until: + if not PairLocks.is_pair_locked(pair, until): + PairLocks.lock_pair(pair, until, reason, now=now) + result = True + return result diff --git a/freqtrade/plugins/protections/__init__.py b/freqtrade/plugins/protections/__init__.py new file mode 100644 index 000000000..936355052 --- /dev/null +++ b/freqtrade/plugins/protections/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa: F401 +from freqtrade.plugins.protections.iprotection import IProtection, ProtectionReturn diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py new file mode 100644 index 000000000..2d7d7b4c7 --- /dev/null +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -0,0 +1,72 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn + + +logger = logging.getLogger(__name__) + + +class CooldownPeriod(IProtection): + + has_global_stop: bool = False + has_local_stop: bool = True + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + def _reason(self) -> str: + """ + LockReason to use + """ + return (f'Cooldown period for {self.stop_duration_str}.') + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Cooldown period of {self.stop_duration_str}.") + + def _cooldown_period(self, pair: str, date_now: datetime, ) -> ProtectionReturn: + """ + Get last trade for this pair + """ + look_back_until = date_now - timedelta(minutes=self._stop_duration) + # filters = [ + # Trade.is_open.is_(False), + # Trade.close_date > look_back_until, + # Trade.pair == pair, + # ] + # trade = Trade.get_trades(filters).first() + trades = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) + if trades: + # Get latest trade + trade = sorted(trades, key=lambda t: t.close_date)[-1] + self.log_once(f"Cooldown for {pair} for {self.stop_duration_str}.", logger.info) + until = self.calculate_lock_end([trade], self._stop_duration) + + return True, until, self._reason() + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + # Not implemented for cooldown period. + return False, None, None + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return self._cooldown_period(pair, date_now) diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py new file mode 100644 index 000000000..684bf6cd3 --- /dev/null +++ b/freqtrade/plugins/protections/iprotection.py @@ -0,0 +1,107 @@ + +import logging +from abc import ABC, abstractmethod +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Tuple + +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.misc import plural +from freqtrade.mixins import LoggingMixin +from freqtrade.persistence import Trade + + +logger = logging.getLogger(__name__) + +ProtectionReturn = Tuple[bool, Optional[datetime], Optional[str]] + + +class IProtection(LoggingMixin, ABC): + + # Can globally stop the bot + has_global_stop: bool = False + # Can stop trading for one pair + has_local_stop: bool = False + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + self._config = config + self._protection_config = protection_config + tf_in_min = timeframe_to_minutes(config['timeframe']) + if 'stop_duration_candles' in protection_config: + self._stop_duration_candles = protection_config.get('stop_duration_candles', 1) + self._stop_duration = (tf_in_min * self._stop_duration_candles) + else: + self._stop_duration_candles = None + self._stop_duration = protection_config.get('stop_duration', 60) + if 'lookback_period_candles' in protection_config: + self._lookback_period_candles = protection_config.get('lookback_period_candles', 1) + self._lookback_period = tf_in_min * self._lookback_period_candles + else: + self._lookback_period_candles = None + self._lookback_period = protection_config.get('lookback_period', 60) + + LoggingMixin.__init__(self, logger) + + @property + def name(self) -> str: + return self.__class__.__name__ + + @property + def stop_duration_str(self) -> str: + """ + Output configured stop duration in either candles or minutes + """ + if self._stop_duration_candles: + return (f"{self._stop_duration_candles} " + f"{plural(self._stop_duration_candles, 'candle', 'candles')}") + else: + return (f"{self._stop_duration} " + f"{plural(self._stop_duration, 'minute', 'minutes')}") + + @property + def lookback_period_str(self) -> str: + """ + Output configured lookback period in either candles or minutes + """ + if self._lookback_period_candles: + return (f"{self._lookback_period_candles} " + f"{plural(self._lookback_period_candles, 'candle', 'candles')}") + else: + return (f"{self._lookback_period} " + f"{plural(self._lookback_period, 'minute', 'minutes')}") + + @abstractmethod + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + -> Please overwrite in subclasses + """ + + @abstractmethod + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + """ + + @abstractmethod + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + + @staticmethod + def calculate_lock_end(trades: List[Trade], stop_minutes: int) -> datetime: + """ + Get lock end time + """ + max_date: datetime = max([trade.close_date for trade in trades]) + # comming from Database, tzinfo is not set. + if max_date.tzinfo is None: + max_date = max_date.replace(tzinfo=timezone.utc) + + until = max_date + timedelta(minutes=stop_minutes) + + return until diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py new file mode 100644 index 000000000..9d5ed35b4 --- /dev/null +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -0,0 +1,83 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn + + +logger = logging.getLogger(__name__) + + +class LowProfitPairs(IProtection): + + has_global_stop: bool = False + has_local_stop: bool = True + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + self._trade_limit = protection_config.get('trade_limit', 1) + self._required_profit = protection_config.get('required_profit', 0.0) + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Low Profit Protection, locks pairs with " + f"profit < {self._required_profit} within {self.lookback_period_str}.") + + def _reason(self, profit: float) -> str: + """ + LockReason to use + """ + return (f'{profit} < {self._required_profit} in {self.lookback_period_str}, ' + f'locking for {self.stop_duration_str}.') + + def _low_profit(self, date_now: datetime, pair: str) -> ProtectionReturn: + """ + Evaluate recent trades for pair + """ + look_back_until = date_now - timedelta(minutes=self._lookback_period) + # filters = [ + # Trade.is_open.is_(False), + # Trade.close_date > look_back_until, + # ] + # if pair: + # filters.append(Trade.pair == pair) + + trades = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) + # trades = Trade.get_trades(filters).all() + if len(trades) < self._trade_limit: + # Not enough trades in the relevant period + return False, None, None + + profit = sum(trade.close_profit for trade in trades) + if profit < self._required_profit: + self.log_once( + f"Trading for {pair} stopped due to {profit:.2f} < {self._required_profit} " + f"within {self._lookback_period} minutes.", logger.info) + until = self.calculate_lock_end(trades, self._stop_duration) + + return True, until, self._reason(profit) + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + return False, None, None + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return self._low_profit(date_now, pair=pair) diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py new file mode 100644 index 000000000..d54e6699b --- /dev/null +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -0,0 +1,88 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +import pandas as pd + +from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn + + +logger = logging.getLogger(__name__) + + +class MaxDrawdown(IProtection): + + has_global_stop: bool = True + has_local_stop: bool = False + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + self._trade_limit = protection_config.get('trade_limit', 1) + self._max_allowed_drawdown = protection_config.get('max_allowed_drawdown', 0.0) + # TODO: Implement checks to limit max_drawdown to sensible values + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Max drawdown protection, stop trading if drawdown is > " + f"{self._max_allowed_drawdown} within {self.lookback_period_str}.") + + def _reason(self, drawdown: float) -> str: + """ + LockReason to use + """ + return (f'{drawdown} > {self._max_allowed_drawdown} in {self.lookback_period_str}, ' + f'locking for {self.stop_duration_str}.') + + def _max_drawdown(self, date_now: datetime) -> ProtectionReturn: + """ + Evaluate recent trades for drawdown ... + """ + look_back_until = date_now - timedelta(minutes=self._lookback_period) + + trades = Trade.get_trades_proxy(is_open=False, close_date=look_back_until) + + trades_df = pd.DataFrame([trade.to_json() for trade in trades]) + + if len(trades) < self._trade_limit: + # Not enough trades in the relevant period + return False, None, None + + # Drawdown is always positive + try: + drawdown, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') + except ValueError: + return False, None, None + + if drawdown > self._max_allowed_drawdown: + self.log_once( + f"Trading stopped due to Max Drawdown {drawdown:.2f} < {self._max_allowed_drawdown}" + f" within {self.lookback_period_str}.", logger.info) + until = self.calculate_lock_end(trades, self._stop_duration) + + return True, until, self._reason(drawdown) + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + return self._max_drawdown(date_now) + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return False, None, None diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py new file mode 100644 index 000000000..193907ddc --- /dev/null +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -0,0 +1,86 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn +from freqtrade.strategy.interface import SellType + + +logger = logging.getLogger(__name__) + + +class StoplossGuard(IProtection): + + has_global_stop: bool = True + has_local_stop: bool = True + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + self._trade_limit = protection_config.get('trade_limit', 10) + self._disable_global_stop = protection_config.get('only_per_pair', False) + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Frequent Stoploss Guard, {self._trade_limit} stoplosses " + f"within {self.lookback_period_str}.") + + def _reason(self) -> str: + """ + LockReason to use + """ + return (f'{self._trade_limit} stoplosses in {self._lookback_period} min, ' + f'locking for {self._stop_duration} min.') + + def _stoploss_guard(self, date_now: datetime, pair: str = None) -> ProtectionReturn: + """ + Evaluate recent trades + """ + look_back_until = date_now - timedelta(minutes=self._lookback_period) + # filters = [ + # Trade.is_open.is_(False), + # Trade.close_date > look_back_until, + # or_(Trade.sell_reason == SellType.STOP_LOSS.value, + # and_(Trade.sell_reason == SellType.TRAILING_STOP_LOSS.value, + # Trade.close_profit < 0)) + # ] + # if pair: + # filters.append(Trade.pair == pair) + # trades = Trade.get_trades(filters).all() + + trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) + trades = [trade for trade in trades1 if str(trade.sell_reason) == SellType.STOP_LOSS.value + or (str(trade.sell_reason) == SellType.TRAILING_STOP_LOSS.value + and trade.close_profit < 0)] + + if len(trades) > self._trade_limit: + self.log_once(f"Trading stopped due to {self._trade_limit} " + f"stoplosses within {self._lookback_period} minutes.", logger.info) + until = self.calculate_lock_end(trades, self._stop_duration) + return True, until, self._reason() + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + if self._disable_global_stop: + return False, None, None + return self._stoploss_guard(date_now, None) + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return self._stoploss_guard(date_now, pair) diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index 8f79349fe..ef24bf481 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -1,6 +1,13 @@ -from freqtrade.resolvers.iresolver import IResolver # noqa: F401 -from freqtrade.resolvers.exchange_resolver import ExchangeResolver # noqa: F401 +# flake8: noqa: F401 +# isort: off +from freqtrade.resolvers.iresolver import IResolver +from freqtrade.resolvers.exchange_resolver import ExchangeResolver +# isort: on # Don't import HyperoptResolver to avoid loading the whole Optimize tree -# from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver # noqa: F401 -from freqtrade.resolvers.pairlist_resolver import PairListResolver # noqa: F401 -from freqtrade.resolvers.strategy_resolver import StrategyResolver # noqa: F401 +# from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver +from freqtrade.resolvers.pairlist_resolver import PairListResolver +from freqtrade.resolvers.protection_resolver import ProtectionResolver +from freqtrade.resolvers.strategy_resolver import StrategyResolver + + + diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index 2b6a731a9..ed6715d15 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -3,10 +3,11 @@ This module loads custom exchanges """ import logging -from freqtrade.exchange import Exchange, MAP_EXCHANGE_CHILDCLASS import freqtrade.exchange as exchanges +from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, Exchange from freqtrade.resolvers import IResolver + logger = logging.getLogger(__name__) diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index ddf461252..8327a4d13 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -7,12 +7,13 @@ import logging from pathlib import Path from typing import Dict -from freqtrade.constants import DEFAULT_HYPEROPT_LOSS, USERPATH_HYPEROPTS +from freqtrade.constants import HYPEROPT_LOSS_BUILTIN, USERPATH_HYPEROPTS from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt_interface import IHyperOpt from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss from freqtrade.resolvers import IResolver + logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ class HyperOptResolver(IResolver): object_type = IHyperOpt object_type_str = "Hyperopt" user_subdir = USERPATH_HYPEROPTS - initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() + initial_search_path = None @staticmethod def load_hyperopt(config: Dict) -> IHyperOpt: @@ -42,14 +43,14 @@ class HyperOptResolver(IResolver): extra_dir=config.get('hyperopt_path')) if not hasattr(hyperopt, 'populate_indicators'): - logger.warning("Hyperopt class does not provide populate_indicators() method. " - "Using populate_indicators from the strategy.") + logger.info("Hyperopt class does not provide populate_indicators() method. " + "Using populate_indicators from the strategy.") if not hasattr(hyperopt, 'populate_buy_trend'): - logger.warning("Hyperopt class does not provide populate_buy_trend() method. " - "Using populate_buy_trend from the strategy.") + logger.info("Hyperopt class does not provide populate_buy_trend() method. " + "Using populate_buy_trend from the strategy.") if not hasattr(hyperopt, 'populate_sell_trend'): - logger.warning("Hyperopt class does not provide populate_sell_trend() method. " - "Using populate_sell_trend from the strategy.") + logger.info("Hyperopt class does not provide populate_sell_trend() method. " + "Using populate_sell_trend from the strategy.") return hyperopt @@ -69,19 +70,19 @@ class HyperOptLossResolver(IResolver): :param config: configuration dictionary """ - # Verify the hyperopt_loss is in the configuration, otherwise fallback to the - # default hyperopt loss - hyperoptloss_name = config.get('hyperopt_loss') or DEFAULT_HYPEROPT_LOSS - + hyperoptloss_name = config.get('hyperopt_loss') + if not hyperoptloss_name: + raise OperationalException( + "No Hyperopt loss set. Please use `--hyperopt-loss` to " + "specify the Hyperopt-Loss class to use.\n" + f"Built-in Hyperopt-loss-functions are: {', '.join(HYPEROPT_LOSS_BUILTIN)}" + ) hyperoptloss = HyperOptLossResolver.load_object(hyperoptloss_name, config, kwargs={}, extra_dir=config.get('hyperopt_path')) - # Assign ticker_interval to be used in hyperopt - hyperoptloss.__class__.ticker_interval = str(config['ticker_interval']) + # Assign timeframe to be used in hyperopt + hyperoptloss.__class__.ticker_interval = str(config['timeframe']) + hyperoptloss.__class__.timeframe = str(config['timeframe']) - if not hasattr(hyperoptloss, 'hyperopt_loss_function'): - raise OperationalException( - f"Found HyperoptLoss class {hyperoptloss_name} does not " - "implement `hyperopt_loss_function`.") return hyperoptloss diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 52d944f2c..37cfd70e6 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -11,6 +11,7 @@ from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union from freqtrade.exceptions import OperationalException + logger = logging.getLogger(__name__) @@ -50,7 +51,8 @@ class IResolver: :param object_name: Class name of the object :param enum_failed: If True, will return None for modules which fail. Otherwise, failing modules are skipped. - :return: generator containing matching objects + :return: generator containing tuple of matching objects + Tuple format: [Object, source] """ # Generate spec based on absolute path @@ -59,21 +61,23 @@ class IResolver: module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints - except (ModuleNotFoundError, SyntaxError) as err: + except (ModuleNotFoundError, SyntaxError, ImportError) as err: # Catch errors in case a specific module is not installed logger.warning(f"Could not import {module_path} due to '{err}'") if enum_failed: return iter([None]) valid_objects_gen = ( - obj for name, obj in inspect.getmembers(module, inspect.isclass) - if ((object_name is None or object_name == name) and - issubclass(obj, cls.object_type) and obj is not cls.object_type) + (obj, inspect.getsource(module)) for + name, obj in inspect.getmembers( + module, inspect.isclass) if ((object_name is None or object_name == name) + and issubclass(obj, cls.object_type) + and obj is not cls.object_type) ) return valid_objects_gen @classmethod - def _search_object(cls, directory: Path, object_name: str + def _search_object(cls, directory: Path, *, object_name: str, add_source: bool = False ) -> Union[Tuple[Any, Path], Tuple[None, None]]: """ Search for the objectname in the given directory @@ -92,11 +96,14 @@ class IResolver: obj = next(cls._get_valid_object(module_path, object_name), None) if obj: - return (obj, module_path) + obj[0].__file__ = str(entry) + if add_source: + obj[0].__source__ = obj[1] + return (obj[0], module_path) return (None, None) @classmethod - def _load_object(cls, paths: List[Path], object_name: str, + def _load_object(cls, paths: List[Path], *, object_name: str, add_source: bool = False, kwargs: dict = {}) -> Optional[Any]: """ Try to load object from path list. @@ -105,7 +112,8 @@ class IResolver: for _path in paths: try: (module, module_path) = cls._search_object(directory=_path, - object_name=object_name) + object_name=object_name, + add_source=add_source) if module: logger.info( f"Using resolved {cls.object_type.__name__.lower()[1:]} {object_name} " @@ -117,7 +125,7 @@ class IResolver: return None @classmethod - def load_object(cls, object_name: str, config: dict, kwargs: dict, + def load_object(cls, object_name: str, config: dict, *, kwargs: dict, extra_dir: Optional[str] = None) -> Any: """ Search and loads the specified object as configured in hte child class. @@ -132,10 +140,10 @@ class IResolver: user_subdir=cls.user_subdir, extra_dir=extra_dir) - pairlist = cls._load_object(paths=abs_paths, object_name=object_name, - kwargs=kwargs) - if pairlist: - return pairlist + found_object = cls._load_object(paths=abs_paths, object_name=object_name, + kwargs=kwargs) + if found_object: + return found_object raise OperationalException( f"Impossible to load {cls.object_type_str} '{object_name}'. This class does not exist " "or contains Python code errors." @@ -163,8 +171,8 @@ class IResolver: for obj in cls._get_valid_object(module_path, object_name=None, enum_failed=enum_failed): objects.append( - {'name': obj.__name__ if obj is not None else '', - 'class': obj, + {'name': obj[0].__name__ if obj is not None else '', + 'class': obj[0] if obj is not None else None, 'location': entry, }) return objects diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 77db74084..72a3cc1dd 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -6,9 +6,10 @@ This module load custom pairlists import logging from pathlib import Path -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.resolvers import IResolver + logger = logging.getLogger(__name__) @@ -19,7 +20,7 @@ class PairListResolver(IResolver): object_type = IPairList object_type_str = "Pairlist" user_subdir = None - initial_search_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() + initial_search_path = Path(__file__).parent.parent.joinpath('plugins/pairlist').resolve() @staticmethod def load_pairlist(pairlist_name: str, exchange, pairlistmanager, diff --git a/freqtrade/resolvers/protection_resolver.py b/freqtrade/resolvers/protection_resolver.py new file mode 100644 index 000000000..c54ae1011 --- /dev/null +++ b/freqtrade/resolvers/protection_resolver.py @@ -0,0 +1,37 @@ +""" +This module load custom pairlists +""" +import logging +from pathlib import Path +from typing import Dict + +from freqtrade.plugins.protections import IProtection +from freqtrade.resolvers import IResolver + + +logger = logging.getLogger(__name__) + + +class ProtectionResolver(IResolver): + """ + This class contains all the logic to load custom PairList class + """ + object_type = IProtection + object_type_str = "Protection" + user_subdir = None + initial_search_path = Path(__file__).parent.parent.joinpath('plugins/protections').resolve() + + @staticmethod + def load_protection(protection_name: str, config: Dict, protection_config: Dict) -> IProtection: + """ + Load the protection with protection_name + :param protection_name: Classname of the pairlist + :param config: configuration dictionary + :param protection_config: Configuration dedicated to this pairlist + :return: initialized Protection class + """ + return ProtectionResolver.load_object(protection_name, config, + kwargs={'config': config, + 'protection_config': protection_config, + }, + ) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index abd6a4195..73af00fee 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -11,12 +11,12 @@ from inspect import getfullargspec from pathlib import Path from typing import Any, Dict, Optional -from freqtrade.constants import (REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, - USERPATH_STRATEGIES) +from freqtrade.constants import REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException from freqtrade.resolvers import IResolver from freqtrade.strategy.interface import IStrategy + logger = logging.getLogger(__name__) @@ -50,11 +50,19 @@ class StrategyResolver(IResolver): if 'ask_strategy' not in config: config['ask_strategy'] = {} + if hasattr(strategy, 'ticker_interval') and not hasattr(strategy, 'timeframe'): + # Assign ticker_interval to timeframe to keep compatibility + if 'timeframe' not in config: + logger.warning( + "DEPRECATED: Please migrate to using 'timeframe' instead of 'ticker_interval'." + ) + strategy.timeframe = strategy.ticker_interval + # Set attributes # Check if we need to override configuration # (Attribute name, default, subkey) attributes = [("minimal_roi", {"0": 10.0}, None), - ("ticker_interval", None, None), + ("timeframe", None, None), ("stoploss", None, None), ("trailing_stop", None, None), ("trailing_stop_positive", None, None), @@ -87,11 +95,7 @@ class StrategyResolver(IResolver): elif attribute in config: logger.info("Strategy using %s: %s", attribute, config[attribute]) - # Sort and apply type conversions - strategy.minimal_roi = OrderedDict(sorted( - {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), - key=lambda t: t[0])) - strategy.stoploss = float(strategy.stoploss) + StrategyResolver._normalize_attributes(strategy) StrategyResolver._strategy_sanity_validations(strategy) return strategy @@ -120,6 +124,24 @@ class StrategyResolver(IResolver): setattr(strategy, attribute, default) config[attribute] = default + @staticmethod + def _normalize_attributes(strategy: IStrategy) -> IStrategy: + """ + Normalize attributes to have the correct type. + """ + # Assign deprecated variable - to not break users code relying on this. + if hasattr(strategy, 'timeframe'): + strategy.ticker_interval = strategy.timeframe + + # Sort and apply type conversions + if hasattr(strategy, 'minimal_roi'): + strategy.minimal_roi = OrderedDict(sorted( + {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), + key=lambda t: t[0])) + if hasattr(strategy, 'stoploss'): + strategy.stoploss = float(strategy.stoploss) + return strategy + @staticmethod def _strategy_sanity_validations(strategy): if not all(k in strategy.order_types for k in REQUIRED_ORDERTYPES): @@ -163,7 +185,9 @@ class StrategyResolver(IResolver): strategy = StrategyResolver._load_object(paths=abs_paths, object_name=strategy_name, - kwargs={'config': config}) + add_source=True, + kwargs={'config': config}, + ) if strategy: strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index 31c854f82..88978519b 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -1,2 +1,3 @@ -from .rpc import RPC, RPCMessageType, RPCException # noqa -from .rpc_manager import RPCManager # noqa +# flake8: noqa: F401 +from .rpc import RPC, RPCException, RPCMessageType +from .rpc_manager import RPCManager diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 9d0899ccd..31e7f3ff2 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -1,37 +1,43 @@ import logging import threading +from copy import deepcopy from datetime import date, datetime from ipaddress import IPv4Address +from pathlib import Path from typing import Any, Callable, Dict from arrow import Arrow from flask import Flask, jsonify, request from flask.json import JSONEncoder from flask_cors import CORS -from flask_jwt_extended import (JWTManager, create_access_token, - create_refresh_token, get_jwt_identity, - jwt_refresh_token_required, +from flask_jwt_extended import (JWTManager, create_access_token, create_refresh_token, + get_jwt_identity, jwt_refresh_token_required, verify_jwt_in_request_optional) from werkzeug.security import safe_str_cmp from werkzeug.serving import make_server from freqtrade.__init__ import __version__ +from freqtrade.constants import DATETIME_PRINT_FORMAT, USERPATH_STRATEGIES +from freqtrade.exceptions import OperationalException +from freqtrade.persistence import Trade +from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.rpc.rpc import RPC, RPCException + logger = logging.getLogger(__name__) BASE_URI = "/api/v1" -class ArrowJSONEncoder(JSONEncoder): +class FTJSONEncoder(JSONEncoder): def default(self, obj): try: if isinstance(obj, Arrow): return obj.for_json() + elif isinstance(obj, datetime): + return obj.strftime(DATETIME_PRINT_FORMAT) elif isinstance(obj, date): return obj.strftime("%Y-%m-%d") - elif isinstance(obj, datetime): - return obj.strftime("%Y-%m-%d %H:%M:%S") iterable = iter(obj) except TypeError: pass @@ -55,7 +61,7 @@ def require_login(func: Callable[[Any, Any], Any]): # Type should really be Callable[[ApiServer], Any], but that will create a circular dependency -def rpc_catch_errors(func: Callable[[Any], Any]): +def rpc_catch_errors(func: Callable[..., Any]): def func_wrapper(obj, *args, **kwargs): @@ -68,6 +74,11 @@ def rpc_catch_errors(func: Callable[[Any], Any]): return func_wrapper +def shutdown_session(exception=None): + # Remove scoped session + Trade.session.remove() + + class ApiServer(RPC): """ This class runs api server and provides rpc.rpc functionality to it @@ -87,10 +98,11 @@ class ApiServer(RPC): """ super().__init__(freqtrade) - self._config = freqtrade.config self.app = Flask(__name__) self._cors = CORS(self.app, - resources={r"/api/*": {"supports_credentials": True, }} + resources={r"/api/*": { + "supports_credentials": True, + "origins": self._config['api_server'].get('CORS_origins', [])}} ) # Setup the Flask-JWT-Extended extension @@ -98,11 +110,16 @@ class ApiServer(RPC): 'jwt_secret_key', 'super-secret') self.jwt = JWTManager(self.app) - self.app.json_encoder = ArrowJSONEncoder + self.app.json_encoder = FTJSONEncoder + + self.app.teardown_appcontext(shutdown_session) # Register application handling self.register_rest_rpc_urls() + if self._config.get('fiat_display_currency', None): + self._fiat_converter = CryptoToFiatConverter() + thread = threading.Thread(target=self.run, daemon=True) thread.start() @@ -145,16 +162,12 @@ class ApiServer(RPC): """ pass - def rest_dump(self, return_value): - """ Helper function to jsonify object for a webserver """ - return jsonify(return_value) - - def rest_error(self, error_msg): - return jsonify({"error": error_msg}), 502 + def rest_error(self, error_msg, error_code=502): + return jsonify({"error": error_msg}), error_code def register_rest_rpc_urls(self): """ - Registers flask app URLs that are calls to functonality in rpc.rpc. + Registers flask app URLs that are calls to functionality in rpc.rpc. First two arguments passed are /URL and 'Label' Label can be used as a shortcut when refactoring @@ -172,16 +185,20 @@ class ApiServer(RPC): self.app.add_url_rule(f'{BASE_URI}/stop', 'stop', view_func=self._stop, methods=['POST']) self.app.add_url_rule(f'{BASE_URI}/stopbuy', 'stopbuy', view_func=self._stopbuy, methods=['POST']) - self.app.add_url_rule(f'{BASE_URI}/reload_conf', 'reload_conf', - view_func=self._reload_conf, methods=['POST']) + self.app.add_url_rule(f'{BASE_URI}/reload_config', 'reload_config', + view_func=self._reload_config, methods=['POST']) # Info commands self.app.add_url_rule(f'{BASE_URI}/balance', 'balance', view_func=self._balance, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/count', 'count', view_func=self._count, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/locks', 'locks', view_func=self._locks, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/daily', 'daily', view_func=self._daily, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/edge', 'edge', view_func=self._edge, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/logs', 'log', view_func=self._get_logs, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/profit', 'profit', view_func=self._profit, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/stats', 'stats', + view_func=self._stats, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/performance', 'performance', view_func=self._performance, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/status', 'status', @@ -194,6 +211,22 @@ class ApiServer(RPC): view_func=self._ping, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/trades', 'trades', view_func=self._trades, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/trades/', 'trades_delete', + view_func=self._trades_delete, methods=['DELETE']) + + self.app.add_url_rule(f'{BASE_URI}/pair_candles', 'pair_candles', + view_func=self._analysed_candles, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/pair_history', 'pair_history', + view_func=self._analysed_history, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/plot_config', 'plot_config', + view_func=self._plot_config, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/strategies', 'strategies', + view_func=self._list_strategies, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/strategy/', 'strategy', + view_func=self._get_strategy, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/available_pairs', 'pairs', + view_func=self._list_available_pairs, methods=['GET']) + # Combined actions and infos self.app.add_url_rule(f'{BASE_URI}/blacklist', 'blacklist', view_func=self._blacklist, methods=['GET', 'POST']) @@ -204,15 +237,12 @@ class ApiServer(RPC): self.app.add_url_rule(f'{BASE_URI}/forcesell', 'forcesell', view_func=self._forcesell, methods=['POST']) - # TODO: Implement the following - # help (?) - @require_login def page_not_found(self, error): """ Return "404 not found", 404. """ - return self.rest_dump({ + return jsonify({ 'status': 'error', 'reason': f"There's no API call for {request.base_url}.", 'code': 404 @@ -232,7 +262,7 @@ class ApiServer(RPC): 'access_token': create_access_token(identity=keystuff), 'refresh_token': create_refresh_token(identity=keystuff), } - return self.rest_dump(ret) + return jsonify(ret) return jsonify({"error": "Unauthorized"}), 401 @@ -247,7 +277,7 @@ class ApiServer(RPC): new_token = create_access_token(identity=current_user, fresh=False) ret = {'access_token': new_token} - return self.rest_dump(ret) + return jsonify(ret) @require_login @rpc_catch_errors @@ -257,7 +287,7 @@ class ApiServer(RPC): Starts TradeThread in bot if stopped. """ msg = self._rpc_start() - return self.rest_dump(msg) + return jsonify(msg) @require_login @rpc_catch_errors @@ -267,7 +297,7 @@ class ApiServer(RPC): Stops TradeThread in bot if running """ msg = self._rpc_stop() - return self.rest_dump(msg) + return jsonify(msg) @require_login @rpc_catch_errors @@ -277,14 +307,14 @@ class ApiServer(RPC): Sets max_open_trades to 0 and gracefully sells all open trades """ msg = self._rpc_stopbuy() - return self.rest_dump(msg) + return jsonify(msg) @rpc_catch_errors def _ping(self): """ - simple poing version + simple ping version """ - return self.rest_dump({"status": "pong"}) + return jsonify({"status": "pong"}) @require_login @rpc_catch_errors @@ -292,7 +322,7 @@ class ApiServer(RPC): """ Prints the bot's version """ - return self.rest_dump({"version": __version__}) + return jsonify({"version": __version__}) @require_login @rpc_catch_errors @@ -300,17 +330,17 @@ class ApiServer(RPC): """ Prints the bot's version """ - return self.rest_dump(self._rpc_show_config()) + return jsonify(RPC._rpc_show_config(self._config, self._freqtrade.state)) @require_login @rpc_catch_errors - def _reload_conf(self): + def _reload_config(self): """ - Handler for /reload_conf. + Handler for /reload_config. Triggers a config file reload """ - msg = self._rpc_reload_conf() - return self.rest_dump(msg) + msg = self._rpc_reload_config() + return jsonify(msg) @require_login @rpc_catch_errors @@ -320,7 +350,16 @@ class ApiServer(RPC): Returns the number of trades running """ msg = self._rpc_count() - return self.rest_dump(msg) + return jsonify(msg) + + @require_login + @rpc_catch_errors + def _locks(self): + """ + Handler for /locks. + Returns the currently active locks. + """ + return jsonify(self._rpc_locks()) @require_login @rpc_catch_errors @@ -338,7 +377,19 @@ class ApiServer(RPC): self._config.get('fiat_display_currency', '') ) - return self.rest_dump(stats) + return jsonify(stats) + + @require_login + @rpc_catch_errors + def _get_logs(self): + """ + Returns latest logs + get: + param: + limit: Only get a certain number of records + """ + limit = int(request.args.get('limit', 0)) or None + return jsonify(RPC._rpc_get_logs(limit)) @require_login @rpc_catch_errors @@ -349,7 +400,7 @@ class ApiServer(RPC): """ stats = self._rpc_edge() - return self.rest_dump(stats) + return jsonify(stats) @require_login @rpc_catch_errors @@ -365,7 +416,19 @@ class ApiServer(RPC): self._config.get('fiat_display_currency') ) - return self.rest_dump(stats) + return jsonify(stats) + + @require_login + @rpc_catch_errors + def _stats(self): + """ + Handler for /stats. + Returns a Object with "durations" and "sell_reasons" as keys. + """ + + stats = self._rpc_stats() + + return jsonify(stats) @require_login @rpc_catch_errors @@ -378,7 +441,7 @@ class ApiServer(RPC): """ stats = self._rpc_performance() - return self.rest_dump(stats) + return jsonify(stats) @require_login @rpc_catch_errors @@ -390,9 +453,9 @@ class ApiServer(RPC): """ try: results = self._rpc_trade_status() - return self.rest_dump(results) + return jsonify(results) except RPCException: - return self.rest_dump([]) + return jsonify([]) @require_login @rpc_catch_errors @@ -404,7 +467,7 @@ class ApiServer(RPC): """ results = self._rpc_balance(self._config['stake_currency'], self._config.get('fiat_display_currency', '')) - return self.rest_dump(results) + return jsonify(results) @require_login @rpc_catch_errors @@ -416,7 +479,20 @@ class ApiServer(RPC): """ limit = int(request.args.get('limit', 0)) results = self._rpc_trade_history(limit) - return self.rest_dump(results) + return jsonify(results) + + @require_login + @rpc_catch_errors + def _trades_delete(self, tradeid: int): + """ + Handler for DELETE /trades/ endpoint. + Removes the trade from the database (tries to cancel open orders first!) + get: + param: + tradeid: Numeric trade-id assigned to the trade. + """ + result = self._rpc_delete(tradeid) + return jsonify(result) @require_login @rpc_catch_errors @@ -425,7 +501,7 @@ class ApiServer(RPC): Handler for /whitelist. """ results = self._rpc_whitelist() - return self.rest_dump(results) + return jsonify(results) @require_login @rpc_catch_errors @@ -435,7 +511,7 @@ class ApiServer(RPC): """ add = request.json.get("blacklist", None) if request.method == 'POST' else None results = self._rpc_blacklist(add) - return self.rest_dump(results) + return jsonify(results) @require_login @rpc_catch_errors @@ -445,11 +521,13 @@ class ApiServer(RPC): """ asset = request.json.get("pair") price = request.json.get("price", None) + price = float(price) if price is not None else price + trade = self._rpc_forcebuy(asset, price) if trade: - return self.rest_dump(trade.to_json()) + return jsonify(trade.to_json()) else: - return self.rest_dump({"status": f"Error buying pair {asset}."}) + return jsonify({"status": f"Error buying pair {asset}."}) @require_login @rpc_catch_errors @@ -459,4 +537,132 @@ class ApiServer(RPC): """ tradeid = request.json.get("tradeid") results = self._rpc_forcesell(tradeid) - return self.rest_dump(results) + return jsonify(results) + + @require_login + @rpc_catch_errors + def _analysed_candles(self): + """ + Handler for /pair_candles. + Returns the dataframe the bot is using during live/dry operations. + Takes the following get arguments: + get: + parameters: + - pair: Pair + - timeframe: Timeframe to get data for (should be aligned to strategy.timeframe) + - limit: Limit return length to the latest X candles + """ + pair = request.args.get("pair") + timeframe = request.args.get("timeframe") + limit = request.args.get("limit", type=int) + if not pair or not timeframe: + return self.rest_error("Mandatory parameter missing.", 400) + + results = self._rpc_analysed_dataframe(pair, timeframe, limit) + return jsonify(results) + + @require_login + @rpc_catch_errors + def _analysed_history(self): + """ + Handler for /pair_history. + Returns the dataframe of a given timerange + Takes the following get arguments: + get: + parameters: + - pair: Pair + - timeframe: Timeframe to get data for (should be aligned to strategy.timeframe) + - strategy: Strategy to use - Must exist in configured strategy-path! + - timerange: timerange in the format YYYYMMDD-YYYYMMDD (YYYYMMDD- or (-YYYYMMDD)) + are als possible. If omitted uses all available data. + """ + pair = request.args.get("pair") + timeframe = request.args.get("timeframe") + timerange = request.args.get("timerange") + strategy = request.args.get("strategy") + + if not pair or not timeframe or not timerange or not strategy: + return self.rest_error("Mandatory parameter missing.", 400) + + config = deepcopy(self._config) + config.update({ + 'strategy': strategy, + }) + results = RPC._rpc_analysed_history_full(config, pair, timeframe, timerange) + return jsonify(results) + + @require_login + @rpc_catch_errors + def _plot_config(self): + """ + Handler for /plot_config. + """ + return jsonify(self._rpc_plot_config()) + + @require_login + @rpc_catch_errors + def _list_strategies(self): + directory = Path(self._config.get( + 'strategy_path', self._config['user_data_dir'] / USERPATH_STRATEGIES)) + from freqtrade.resolvers.strategy_resolver import StrategyResolver + strategy_objs = StrategyResolver.search_all_objects(directory, False) + strategy_objs = sorted(strategy_objs, key=lambda x: x['name']) + + return jsonify({'strategies': [x['name'] for x in strategy_objs]}) + + @require_login + @rpc_catch_errors + def _get_strategy(self, strategy: str): + """ + Get a single strategy + get: + parameters: + - strategy: Only get this strategy + """ + config = deepcopy(self._config) + from freqtrade.resolvers.strategy_resolver import StrategyResolver + try: + strategy_obj = StrategyResolver._load_strategy(strategy, config, + extra_dir=config.get('strategy_path')) + except OperationalException: + return self.rest_error("Strategy not found.", 404) + + return jsonify({ + 'strategy': strategy_obj.get_strategy_name(), + 'code': strategy_obj.__source__, + }) + + @require_login + @rpc_catch_errors + def _list_available_pairs(self): + """ + Handler for /available_pairs. + Returns an object, with pairs, available pair length and pair_interval combinations + Takes the following get arguments: + get: + parameters: + - stake_currency: Filter on this stake currency + - timeframe: Timeframe to get data for Filter elements to this timeframe + """ + timeframe = request.args.get("timeframe") + stake_currency = request.args.get("stake_currency") + + from freqtrade.data.history import get_datahandler + dh = get_datahandler(self._config['datadir'], self._config.get('dataformat_ohlcv', None)) + + pair_interval = dh.ohlcv_get_available_data(self._config['datadir']) + + if timeframe: + pair_interval = [pair for pair in pair_interval if pair[1] == timeframe] + if stake_currency: + pair_interval = [pair for pair in pair_interval if pair[0].endswith(stake_currency)] + pair_interval = sorted(pair_interval, key=lambda x: x[0]) + + pairs = list({x[0] for x in pair_interval}) + + result = { + 'length': len(pairs), + 'pairs': pairs, + 'pair_interval': pair_interval, + } + return jsonify(result) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 00a170ee3..9b7d62b54 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -6,25 +6,32 @@ from abc import abstractmethod from datetime import date, datetime, timedelta from enum import Enum from math import isnan -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, Union import arrow -from numpy import NAN, mean +from numpy import NAN, int64, mean +from pandas import DataFrame -from freqtrade.exceptions import DependencyException, TemporaryError +from freqtrade.configuration.timerange import TimeRange +from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT +from freqtrade.data.history import load_data +from freqtrade.exceptions import ExchangeError, PricingError +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs +from freqtrade.loggers import bufferHandler from freqtrade.misc import shorten_date -from freqtrade.persistence import Trade +from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State from freqtrade.strategy.interface import SellType + logger = logging.getLogger(__name__) class RPCMessageType(Enum): STATUS_NOTIFICATION = 'status' WARNING_NOTIFICATION = 'warning' - CUSTOM_NOTIFICATION = 'custom' + STARTUP_NOTIFICATION = 'startup' BUY_NOTIFICATION = 'buy' BUY_CANCEL_NOTIFICATION = 'buy_cancel' SELL_NOTIFICATION = 'sell' @@ -33,6 +40,9 @@ class RPCMessageType(Enum): def __repr__(self): return self.value + def __str__(self): + return self.value + class RPCException(Exception): """ @@ -69,6 +79,7 @@ class RPC: :return: None """ self._freqtrade = freqtrade + self._config: Dict[str, Any] = freqtrade.config @property def name(self) -> str: @@ -83,31 +94,36 @@ class RPC: def send_msg(self, msg: Dict[str, str]) -> None: """ Sends a message to all registered rpc modules """ - def _rpc_show_config(self) -> Dict[str, Any]: + @staticmethod + def _rpc_show_config(config, botstate: State) -> Dict[str, Any]: """ Return a dict of config options. Explicitly does NOT return the full config to avoid leakage of sensitive information via rpc. """ - config = self._freqtrade.config val = { 'dry_run': config['dry_run'], 'stake_currency': config['stake_currency'], 'stake_amount': config['stake_amount'], 'max_open_trades': config['max_open_trades'], - 'minimal_roi': config['minimal_roi'].copy(), - 'stoploss': config['stoploss'], - 'trailing_stop': config['trailing_stop'], + 'minimal_roi': config['minimal_roi'].copy() if 'minimal_roi' in config else {}, + 'stoploss': config.get('stoploss'), + 'trailing_stop': config.get('trailing_stop'), 'trailing_stop_positive': config.get('trailing_stop_positive'), 'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset'), 'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached'), - 'ticker_interval': config['ticker_interval'], + 'timeframe': config.get('timeframe'), + 'timeframe_ms': timeframe_to_msecs(config['timeframe'] + ) if 'timeframe' in config else '', + 'timeframe_min': timeframe_to_minutes(config['timeframe'] + ) if 'timeframe' in config else '', 'exchange': config['exchange']['name'], 'strategy': config['strategy'], 'forcebuy_enabled': config.get('forcebuy_enable', False), 'ask_strategy': config.get('ask_strategy', {}), 'bid_strategy': config.get('bid_strategy', {}), - 'state': str(self._freqtrade.state) + 'state': str(botstate), + 'runmode': config['runmode'].value } return val @@ -125,11 +141,11 @@ class RPC: for trade in trades: order = None if trade.open_order_id: - order = self._freqtrade.exchange.get_order(trade.open_order_id, trade.pair) + order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) # calculate profit and send message to user try: current_rate = self._freqtrade.get_sell_rate(trade.pair, False) - except DependencyException: + except (ExchangeError, PricingError): current_rate = NAN current_profit = trade.calc_profit_ratio(current_rate) current_profit_abs = trade.calc_profit(current_rate) @@ -140,19 +156,21 @@ class RPC: stoploss_current_dist = trade.stop_loss - current_rate stoploss_current_dist_ratio = stoploss_current_dist / current_rate - fmt_close_profit = (f'{round(trade.close_profit * 100, 2):.2f}%' - if trade.close_profit is not None else None) trade_dict = trade.to_json() trade_dict.update(dict( base_currency=self._freqtrade.config['stake_currency'], close_profit=trade.close_profit if trade.close_profit is not None else None, - close_profit_pct=fmt_close_profit, current_rate=current_rate, - current_profit=current_profit, - current_profit_pct=round(current_profit * 100, 2), - current_profit_abs=current_profit_abs, + current_profit=current_profit, # Deprectated + current_profit_pct=round(current_profit * 100, 2), # Deprectated + current_profit_abs=current_profit_abs, # Deprectated + profit_ratio=current_profit, + profit_pct=round(current_profit * 100, 2), + profit_abs=current_profit_abs, + stoploss_current_dist=stoploss_current_dist, stoploss_current_dist_ratio=round(stoploss_current_dist_ratio, 8), + stoploss_current_dist_pct=round(stoploss_current_dist_ratio * 100, 2), stoploss_entry_dist=stoploss_entry_dist, stoploss_entry_dist_ratio=round(stoploss_entry_dist_ratio, 8), open_order='({} {} rem={:.8f})'.format( @@ -173,7 +191,7 @@ class RPC: # calculate profit and send message to user try: current_rate = self._freqtrade.get_sell_rate(trade.pair, False) - except DependencyException: + except (PricingError, ExchangeError): current_rate = NAN trade_percent = (100 * trade.calc_profit_ratio(current_rate)) trade_profit = trade.calc_profit(current_rate) @@ -217,24 +235,23 @@ class RPC: Trade.close_date >= profitday, Trade.close_date < (profitday + timedelta(days=1)) ]).order_by(Trade.close_date).all() - curdayprofit = sum(trade.close_profit_abs for trade in trades) + curdayprofit = sum( + trade.close_profit_abs for trade in trades if trade.close_profit_abs is not None) profit_days[profitday] = { - 'amount': f'{curdayprofit:.8f}', + 'amount': curdayprofit, 'trades': len(trades) } data = [ { 'date': key, - 'abs_profit': f'{float(value["amount"]):.8f}', - 'fiat_value': '{value:.3f}'.format( - value=self._fiat_converter.convert_amount( + 'abs_profit': value["amount"], + 'fiat_value': self._fiat_converter.convert_amount( value['amount'], stake_currency, fiat_display_currency ) if self._fiat_converter else 0, - ), - 'trade_count': f'{value["trades"]}', + 'trade_count': value["trades"], } for key, value in profit_days.items() ] @@ -247,9 +264,10 @@ class RPC: def _rpc_trade_history(self, limit: int) -> Dict: """ Returns the X last trades """ if limit > 0: - trades = Trade.get_trades().order_by(Trade.id.desc()).limit(limit) + trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by( + Trade.id.desc()).limit(limit) else: - trades = Trade.get_trades().order_by(Trade.id.desc()).all() + trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(Trade.id.desc()).all() output = [trade.to_json() for trade in trades] @@ -258,6 +276,39 @@ class RPC: "trades_count": len(output) } + def _rpc_stats(self) -> Dict[str, Any]: + """ + Generate generic stats for trades in database + """ + def trade_win_loss(trade): + if trade.close_profit > 0: + return 'wins' + elif trade.close_profit < 0: + return 'losses' + else: + return 'draws' + trades = trades = Trade.get_trades([Trade.is_open.is_(False)]) + # Sell reason + sell_reasons = {} + for trade in trades: + if trade.sell_reason not in sell_reasons: + sell_reasons[trade.sell_reason] = {'wins': 0, 'losses': 0, 'draws': 0} + sell_reasons[trade.sell_reason][trade_win_loss(trade)] += 1 + + # Duration + dur: Dict[str, List[int]] = {'wins': [], 'draws': [], 'losses': []} + for trade in trades: + if trade.close_date is not None and trade.open_date is not None: + trade_dur = (trade.close_date - trade.open_date).total_seconds() + dur[trade_win_loss(trade)].append(trade_dur) + + wins_dur = sum(dur['wins']) / len(dur['wins']) if len(dur['wins']) > 0 else 'N/A' + draws_dur = sum(dur['draws']) / len(dur['draws']) if len(dur['draws']) > 0 else 'N/A' + losses_dur = sum(dur['losses']) / len(dur['losses']) if len(dur['losses']) > 0 else 'N/A' + + durations = {'wins': wins_dur, 'draws': draws_dur, 'losses': losses_dur} + return {'sell_reasons': sell_reasons, 'durations': durations} + def _rpc_trade_statistics( self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: """ Returns cumulative profit statistics """ @@ -268,6 +319,8 @@ class RPC: profit_closed_coin = [] profit_closed_ratio = [] durations = [] + winning_trades = 0 + losing_trades = 0 for trade in trades: current_rate: float = 0.0 @@ -281,11 +334,15 @@ class RPC: profit_ratio = trade.close_profit profit_closed_coin.append(trade.close_profit_abs) profit_closed_ratio.append(profit_ratio) + if trade.close_profit >= 0: + winning_trades += 1 + else: + losing_trades += 1 else: # Get current rate try: current_rate = self._freqtrade.get_sell_rate(trade.pair, False) - except DependencyException: + except (PricingError, ExchangeError): current_rate = NAN profit_ratio = trade.calc_profit_ratio(rate=current_rate) @@ -343,6 +400,8 @@ class RPC: 'avg_duration': str(timedelta(seconds=sum(durations) / num)).split('.')[0], 'best_pair': best_pair[0] if best_pair else '', 'best_rate': round(best_pair[1] * 100, 2) if best_pair else 0, + 'winning_trades': winning_trades, + 'losing_trades': losing_trades, } def _rpc_balance(self, stake_currency: str, fiat_display_currency: str) -> Dict: @@ -351,7 +410,7 @@ class RPC: total = 0.0 try: tickers = self._freqtrade.exchange.get_tickers() - except (TemporaryError, DependencyException): + except (ExchangeError): raise RPCException('Error getting current tickers.') self._freqtrade.wallets.update(require_update=False) @@ -372,7 +431,7 @@ class RPC: if pair.startswith(stake_currency): rate = 1.0 / rate est_stake = rate * balance.total - except (TemporaryError, DependencyException): + except (ExchangeError): logger.warning(f" Could not get rate for pair {coin}.") continue total = total + (est_stake or 0) @@ -418,10 +477,10 @@ class RPC: return {'status': 'already stopped'} - def _rpc_reload_conf(self) -> Dict[str, str]: - """ Handler for reload_conf. """ - self._freqtrade.state = State.RELOAD_CONF - return {'status': 'reloading config ...'} + def _rpc_reload_config(self) -> Dict[str, str]: + """ Handler for reload_config. """ + self._freqtrade.state = State.RELOAD_CONFIG + return {'status': 'Reloading config ...'} def _rpc_stopbuy(self) -> Dict[str, str]: """ @@ -431,7 +490,7 @@ class RPC: # Set 'max_open_trades' to 0 self._freqtrade.config['max_open_trades'] = 0 - return {'status': 'No more buy will occur from now. Run /reload_conf to reset.'} + return {'status': 'No more buy will occur from now. Run /reload_config to reset.'} def _rpc_forcesell(self, trade_id: str) -> Dict[str, str]: """ @@ -440,29 +499,22 @@ class RPC: """ def _exec_forcesell(trade: Trade) -> None: # Check if there is there is an open order + fully_canceled = False if trade.open_order_id: - order = self._freqtrade.exchange.get_order(trade.open_order_id, trade.pair) + order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) - # Cancel open LIMIT_BUY orders and close trade - if order and order['status'] == 'open' \ - and order['type'] == 'limit' \ - and order['side'] == 'buy': - self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair) - trade.close(order.get('price') or trade.open_rate) - # Do the best effort, if we don't know 'filled' amount, don't try selling - if order['filled'] is None: - return - trade.amount = order['filled'] + if order['side'] == 'buy': + fully_canceled = self._freqtrade.handle_cancel_buy( + trade, order, CANCEL_REASON['FORCE_SELL']) - # Ignore trades with an attached LIMIT_SELL order - if order and order['status'] == 'open' \ - and order['type'] == 'limit' \ - and order['side'] == 'sell': - return + if order['side'] == 'sell': + # Cancel order - so it is placed anew with a fresh price. + self._freqtrade.handle_cancel_sell(trade, order, CANCEL_REASON['FORCE_SELL']) - # Get current rate and execute sell - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) - self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL) + if not fully_canceled: + # Get current rate and execute sell + current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + self._freqtrade.execute_sell(trade, current_rate, SellType.FORCE_SELL) # ---- EOF def _exec_forcesell ---- if self._freqtrade.state != State.RUNNING: @@ -506,11 +558,11 @@ class RPC: stake_currency = self._freqtrade.config.get('stake_currency') if not self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency: raise RPCException( - f'Wrong pair selected. Please pairs with stake {stake_currency} pairs only') + f'Wrong pair selected. Only pairs with stake-currency {stake_currency} allowed.') # check if valid pair # check if pair already has an open pair - trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first() + trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() if trade: raise RPCException(f'position for {pair} already open - id: {trade.id}') @@ -519,11 +571,50 @@ class RPC: # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price): - trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair.is_(pair)]).first() + trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() return trade else: return None + def _rpc_delete(self, trade_id: int) -> Dict[str, Union[str, int]]: + """ + Handler for delete . + Delete the given trade and close eventually existing open orders. + """ + with self._freqtrade._sell_lock: + c_count = 0 + trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first() + if not trade: + logger.warning('delete trade: Invalid argument received') + raise RPCException('invalid argument') + + # Try cancelling regular order if that exists + if trade.open_order_id: + try: + self._freqtrade.exchange.cancel_order(trade.open_order_id, trade.pair) + c_count += 1 + except (ExchangeError): + pass + + # cancel stoploss on exchange ... + if (self._freqtrade.strategy.order_types.get('stoploss_on_exchange') + and trade.stoploss_order_id): + try: + self._freqtrade.exchange.cancel_stoploss_order(trade.stoploss_order_id, + trade.pair) + c_count += 1 + except (ExchangeError): + pass + + trade.delete() + self._freqtrade.wallets.update() + return { + 'result': 'success', + 'trade_id': trade_id, + 'result_msg': f'Deleted trade {trade_id}. Closed {c_count} open orders.', + 'cancel_order_count': c_count, + } + def _rpc_performance(self) -> List[Dict[str, Any]]: """ Handler for performance. @@ -546,6 +637,15 @@ class RPC: 'total_stake': sum((trade.open_rate * trade.amount) for trade in trades) } + def _rpc_locks(self) -> Dict[str, Any]: + """ Returns the current locks""" + + locks = PairLocks.get_pair_locks(None) + return { + 'lock_count': len(locks), + 'locks': [lock.to_json() for lock in locks] + } + def _rpc_whitelist(self) -> Dict: """ Returns the currently active whitelist""" res = {'method': self._freqtrade.pairlists.name_list, @@ -579,8 +679,106 @@ class RPC: } return res + @staticmethod + def _rpc_get_logs(limit: Optional[int]) -> Dict[str, Any]: + """Returns the last X logs""" + if limit: + buffer = bufferHandler.buffer[-limit:] + else: + buffer = bufferHandler.buffer + records = [[datetime.fromtimestamp(r.created).strftime(DATETIME_PRINT_FORMAT), + r.created * 1000, r.name, r.levelname, + r.message + ('\n' + r.exc_text if r.exc_text else '')] + for r in buffer] + + # Log format: + # [logtime-formatted, logepoch, logger-name, loglevel, message \n + exception] + # e.g. ["2020-08-27 11:35:01", 1598520901097.9397, + # "freqtrade.worker", "INFO", "Starting worker develop"] + + return {'log_count': len(records), 'logs': records} + def _rpc_edge(self) -> List[Dict[str, Any]]: """ Returns information related to Edge """ if not self._freqtrade.edge: raise RPCException('Edge is not enabled.') return self._freqtrade.edge.accepted_pairs() + + @staticmethod + def _convert_dataframe_to_dict(strategy: str, pair: str, timeframe: str, dataframe: DataFrame, + last_analyzed: datetime) -> Dict[str, Any]: + has_content = len(dataframe) != 0 + buy_signals = 0 + sell_signals = 0 + if has_content: + + dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000 + # Move open to seperate column when signal for easy plotting + if 'buy' in dataframe.columns: + buy_mask = (dataframe['buy'] == 1) + buy_signals = int(buy_mask.sum()) + dataframe.loc[buy_mask, '_buy_signal_open'] = dataframe.loc[buy_mask, 'open'] + if 'sell' in dataframe.columns: + sell_mask = (dataframe['sell'] == 1) + sell_signals = int(sell_mask.sum()) + dataframe.loc[sell_mask, '_sell_signal_open'] = dataframe.loc[sell_mask, 'open'] + dataframe = dataframe.replace({NAN: None}) + + res = { + 'pair': pair, + 'timeframe': timeframe, + 'timeframe_ms': timeframe_to_msecs(timeframe), + 'strategy': strategy, + 'columns': list(dataframe.columns), + 'data': dataframe.values.tolist(), + 'length': len(dataframe), + 'buy_signals': buy_signals, + 'sell_signals': sell_signals, + 'last_analyzed': last_analyzed, + 'last_analyzed_ts': int(last_analyzed.timestamp()), + 'data_start': '', + 'data_start_ts': 0, + 'data_stop': '', + 'data_stop_ts': 0, + } + if has_content: + res.update({ + 'data_start': str(dataframe.iloc[0]['date']), + 'data_start_ts': int(dataframe.iloc[0]['__date_ts']), + 'data_stop': str(dataframe.iloc[-1]['date']), + 'data_stop_ts': int(dataframe.iloc[-1]['__date_ts']), + }) + return res + + def _rpc_analysed_dataframe(self, pair: str, timeframe: str, limit: int) -> Dict[str, Any]: + + _data, last_analyzed = self._freqtrade.dataprovider.get_analyzed_dataframe( + pair, timeframe) + _data = _data.copy() + if limit: + _data = _data.iloc[-limit:] + return self._convert_dataframe_to_dict(self._freqtrade.config['strategy'], + pair, timeframe, _data, last_analyzed) + + @staticmethod + def _rpc_analysed_history_full(config, pair: str, timeframe: str, + timerange: str) -> Dict[str, Any]: + timerange_parsed = TimeRange.parse_timerange(timerange) + + _data = load_data( + datadir=config.get("datadir"), + pairs=[pair], + timeframe=timeframe, + timerange=timerange_parsed, + data_format=config.get('dataformat_ohlcv', 'json'), + ) + from freqtrade.resolvers.strategy_resolver import StrategyResolver + strategy = StrategyResolver.load_strategy(config) + df_analyzed = strategy.analyze_ticker(_data[pair], {'pair': pair}) + + return RPC._convert_dataframe_to_dict(strategy.get_strategy_name(), pair, timeframe, + df_analyzed, arrow.Arrow.utcnow().datetime) + + def _rpc_plot_config(self) -> Dict[str, Any]: + + return self._freqtrade.strategy.plot_config diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 670275991..c42878f99 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -6,6 +6,7 @@ from typing import Any, Dict, List from freqtrade.rpc import RPC, RPCMessageType + logger = logging.getLogger(__name__) @@ -59,9 +60,9 @@ class RPCManager: try: mod.send_msg(msg) except NotImplementedError: - logger.error(f"Message type {msg['type']} not implemented by handler {mod.name}.") + logger.error(f"Message type '{msg['type']}' not implemented by handler {mod.name}.") - def startup_messages(self, config: Dict[str, Any], pairlist) -> None: + def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None: if config['dry_run']: self.send_msg({ 'type': RPCMessageType.WARNING_NOTIFICATION, @@ -72,20 +73,26 @@ class RPCManager: minimal_roi = config['minimal_roi'] stoploss = config['stoploss'] trailing_stop = config['trailing_stop'] - ticker_interval = config['ticker_interval'] + timeframe = config['timeframe'] exchange_name = config['exchange']['name'] strategy_name = config.get('strategy', '') self.send_msg({ - 'type': RPCMessageType.CUSTOM_NOTIFICATION, + 'type': RPCMessageType.STARTUP_NOTIFICATION, 'status': f'*Exchange:* `{exchange_name}`\n' f'*Stake per trade:* `{stake_amount} {stake_currency}`\n' f'*Minimum ROI:* `{minimal_roi}`\n' f'*{"Trailing " if trailing_stop else ""}Stoploss:* `{stoploss}`\n' - f'*Ticker Interval:* `{ticker_interval}`\n' + f'*Timeframe:* `{timeframe}`\n' f'*Strategy:* `{strategy_name}`' }) self.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, + 'type': RPCMessageType.STARTUP_NOTIFICATION, 'status': f'Searching for {stake_currency} pairs to buy and sell ' f'based on {pairlist.short_desc()}' }) + if len(protections.name_list) > 0: + prots = '\n'.join([p for prot in protections.short_desc() for k, p in prot.items()]) + self.send_msg({ + 'type': RPCMessageType.STARTUP_NOTIFICATION, + 'status': f'Using Protections: \n{prots}' + }) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index eb53fc68f..dddba7457 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,17 +5,23 @@ This module manage Telegram communication """ import json import logging -from typing import Any, Callable, Dict +from datetime import timedelta +from itertools import chain +from typing import Any, Callable, Dict, List, Union +import arrow from tabulate import tabulate -from telegram import ParseMode, ReplyKeyboardMarkup, Update +from telegram import KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CallbackContext, CommandHandler, Updater +from telegram.utils.helpers import escape_markdown from freqtrade.__init__ import __version__ +from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC, RPCException, RPCMessageType from freqtrade.rpc.fiat_convert import CryptoToFiatConverter + logger = logging.getLogger(__name__) logger.debug('Included module rpc.telegram ...') @@ -68,12 +74,50 @@ class Telegram(RPC): """ super().__init__(freqtrade) - self._updater: Updater = None - self._config = freqtrade.config + self._updater: Updater + self._init_keyboard() self._init() if self._config.get('fiat_display_currency', None): self._fiat_converter = CryptoToFiatConverter() + def _init_keyboard(self) -> None: + """ + Validates the keyboard configuration from telegram config + section. + """ + self._keyboard: List[List[Union[str, KeyboardButton]]] = [ + ['/daily', '/profit', '/balance'], + ['/status', '/status table', '/performance'], + ['/count', '/start', '/stop', '/help'] + ] + # do not allow commands with mandatory arguments and critical cmds + # like /forcesell and /forcebuy + # TODO: DRY! - its not good to list all valid cmds here. But otherwise + # this needs refacoring of the whole telegram module (same + # problem in _help()). + valid_keys: List[str] = ['/start', '/stop', '/status', '/status table', + '/trades', '/profit', '/performance', '/daily', + '/stats', '/count', '/locks', '/balance', + '/stopbuy', '/reload_config', '/show_config', + '/logs', '/whitelist', '/blacklist', '/edge', + '/help', '/version'] + + # custom keyboard specified in config.json + cust_keyboard = self._config['telegram'].get('keyboard', []) + if cust_keyboard: + # check for valid shortcuts + invalid_keys = [b for b in chain.from_iterable(cust_keyboard) + if b not in valid_keys] + if len(invalid_keys): + err_msg = ('config.telegram.keyboard: Invalid commands for ' + f'custom Telegram keyboard: {invalid_keys}' + f'\nvalid commands are: {valid_keys}') + raise OperationalException(err_msg) + else: + self._keyboard = cust_keyboard + logger.info('using custom keyboard from ' + f'config.json: {self._keyboard}') + def _init(self) -> None: """ Initializes this module with the given config, @@ -92,14 +136,19 @@ class Telegram(RPC): CommandHandler('stop', self._stop), CommandHandler('forcesell', self._forcesell), CommandHandler('forcebuy', self._forcebuy), + CommandHandler('trades', self._trades), + CommandHandler('delete', self._delete_trade), CommandHandler('performance', self._performance), + CommandHandler('stats', self._stats), CommandHandler('daily', self._daily), CommandHandler('count', self._count), - CommandHandler('reload_conf', self._reload_conf), - CommandHandler('show_config', self._show_config), + CommandHandler('locks', self._locks), + CommandHandler(['reload_config', 'reload_conf'], self._reload_config), + CommandHandler(['show_config', 'show_conf'], self._show_config), CommandHandler('stopbuy', self._stopbuy), CommandHandler('whitelist', self._whitelist), CommandHandler('blacklist', self._blacklist), + CommandHandler('logs', self._logs), CommandHandler('edge', self._edge), CommandHandler('help', self._help), CommandHandler('version', self._version), @@ -127,6 +176,13 @@ class Telegram(RPC): def send_msg(self, msg: Dict[str, Any]) -> None: """ Send a message to telegram channel """ + noti = self._config['telegram'].get('notification_settings', {} + ).get(str(msg['type']), 'on') + if noti == 'off': + logger.info(f"Notification '{msg['type']}' not sent.") + # Notification disabled + return + if msg['type'] == RPCMessageType.BUY_NOTIFICATION: if self._fiat_converter: msg['stake_amount_fiat'] = self._fiat_converter.convert_amount( @@ -146,7 +202,7 @@ class Telegram(RPC): elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: message = ("\N{WARNING SIGN} *{exchange}:* " - "Cancelling Open Buy Order for {pair}".format(**msg)) + "Cancelling open buy Order for {pair}. Reason: {reason}.".format(**msg)) elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: msg['amount'] = round(msg['amount'], 8) @@ -185,13 +241,13 @@ class Telegram(RPC): elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION: message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) - elif msg['type'] == RPCMessageType.CUSTOM_NOTIFICATION: + elif msg['type'] == RPCMessageType.STARTUP_NOTIFICATION: message = '{status}'.format(**msg) else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) - self._send_msg(message) + self._send_msg(message, disable_notification=(noti == 'silent')) def _get_sell_emoji(self, msg): """ @@ -217,7 +273,7 @@ class Telegram(RPC): :return: None """ - if 'table' in context.args: + if context.args and 'table' in context.args: self._status_table(update, context) return @@ -233,20 +289,20 @@ class Telegram(RPC): "*Open Rate:* `{open_rate:.8f}`", "*Close Rate:* `{close_rate}`" if r['close_rate'] else "", "*Current Rate:* `{current_rate:.8f}`", - ("*Close Profit:* `{close_profit_pct}`" - if r['close_profit_pct'] is not None else ""), - "*Current Profit:* `{current_profit_pct:.2f}%`", - - # Adding initial stoploss only if it is different from stoploss - "*Initial Stoploss:* `{initial_stop_loss:.8f}` " + - ("`({initial_stop_loss_pct:.2f}%)`") if ( - r['stop_loss'] != r['initial_stop_loss'] - and r['initial_stop_loss_pct'] is not None) else "", - - # Adding stoploss and stoploss percentage only if it is not None - "*Stoploss:* `{stop_loss:.8f}` " + - ("`({stop_loss_pct:.2f}%)`" if r['stop_loss_pct'] else ""), + ("*Current Profit:* " if r['is_open'] else "*Close Profit: *") + + "`{profit_pct:.2f}%`", ] + if (r['stop_loss_abs'] != r['initial_stop_loss_abs'] + and r['initial_stop_loss_pct'] is not None): + # Adding initial stoploss only if it is different from stoploss + lines.append("*Initial Stoploss:* `{initial_stop_loss_abs:.8f}` " + "`({initial_stop_loss_pct:.2f}%)`") + + # Adding stoploss and stoploss percentage only if it is not None + lines.append("*Stoploss:* `{stop_loss_abs:.8f}` " + + ("`({stop_loss_pct:.2f}%)`" if r['stop_loss_pct'] else "")) + lines.append("*Stoploss distance:* `{stoploss_current_dist:.8f}` " + "`({stoploss_current_dist_pct:.2f}%)`") if r['open_order']: if r['sell_order_status']: lines.append("*Open Order:* `{open_order}` - `{sell_order_status}`") @@ -291,7 +347,7 @@ class Telegram(RPC): stake_cur = self._config['stake_currency'] fiat_disp_cur = self._config.get('fiat_display_currency', '') try: - timescale = int(context.args[0]) + timescale = int(context.args[0]) if context.args else 7 except (TypeError, ValueError, IndexError): timescale = 7 try: @@ -302,8 +358,8 @@ class Telegram(RPC): ) stats_tab = tabulate( [[day['date'], - f"{day['abs_profit']} {stats['stake_currency']}", - f"{day['fiat_value']} {stats['fiat_display_currency']}", + f"{day['abs_profit']:.8f} {stats['stake_currency']}", + f"{day['fiat_value']:.3f} {stats['fiat_display_currency']}", f"{day['trade_count']} trades"] for day in stats['data']], headers=[ 'Day', @@ -366,12 +422,56 @@ class Telegram(RPC): f"∙ `{profit_all_fiat:.3f} {fiat_disp_cur}`\n" f"*Total Trade Count:* `{trade_count}`\n" f"*First Trade opened:* `{first_trade_date}`\n" - f"*Latest Trade opened:* `{latest_trade_date}`") + f"*Latest Trade opened:* `{latest_trade_date}\n`" + f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" + ) if stats['closed_trade_count'] > 0: markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") self._send_msg(markdown_msg) + @authorized_only + def _stats(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /stats + Show stats of recent trades + """ + stats = self._rpc_stats() + + reason_map = { + 'roi': 'ROI', + 'stop_loss': 'Stoploss', + 'trailing_stop_loss': 'Trail. Stop', + 'stoploss_on_exchange': 'Stoploss', + 'sell_signal': 'Sell Signal', + 'force_sell': 'Forcesell', + 'emergency_sell': 'Emergency Sell', + } + sell_reasons_tabulate = [ + [ + reason_map.get(reason, reason), + sum(count.values()), + count['wins'], + count['losses'] + ] for reason, count in stats['sell_reasons'].items() + ] + sell_reasons_msg = tabulate( + sell_reasons_tabulate, + headers=['Sell Reason', 'Sells', 'Wins', 'Losses'] + ) + durations = stats['durations'] + duration_msg = tabulate([ + ['Wins', str(timedelta(seconds=durations['wins'])) + if durations['wins'] != 'N/A' else 'N/A'], + ['Losses', str(timedelta(seconds=durations['losses'])) + if durations['losses'] != 'N/A' else 'N/A'] + ], + headers=['', 'Avg. Duration'] + ) + msg = (f"""```\n{sell_reasons_msg}```\n```\n{duration_msg}```""") + + self._send_msg(msg, ParseMode.MARKDOWN) + @authorized_only def _balance(self, update: Update, context: CallbackContext) -> None: """ Handler for /balance """ @@ -436,15 +536,15 @@ class Telegram(RPC): self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only - def _reload_conf(self, update: Update, context: CallbackContext) -> None: + def _reload_config(self, update: Update, context: CallbackContext) -> None: """ - Handler for /reload_conf. + Handler for /reload_config. Triggers a config file reload :param bot: telegram bot :param update: message update :return: None """ - msg = self._rpc_reload_conf() + msg = self._rpc_reload_config() self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only @@ -469,7 +569,10 @@ class Telegram(RPC): :return: None """ - trade_id = context.args[0] if len(context.args) > 0 else None + trade_id = context.args[0] if context.args and len(context.args) > 0 else None + if not trade_id: + self._send_msg("You must specify a trade-id or 'all'.") + return try: msg = self._rpc_forcesell(trade_id) self._send_msg('Forcesell Result: `{result}`'.format(**msg)) @@ -486,11 +589,68 @@ class Telegram(RPC): :param update: message update :return: None """ + if context.args: + pair = context.args[0] + price = float(context.args[1]) if len(context.args) > 1 else None + try: + self._rpc_forcebuy(pair, price) + except RPCException as e: + self._send_msg(str(e)) - pair = context.args[0] - price = float(context.args[1]) if len(context.args) > 1 else None + @authorized_only + def _trades(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /trades + Returns last n recent trades. + :param bot: telegram bot + :param update: message update + :return: None + """ + stake_cur = self._config['stake_currency'] try: - self._rpc_forcebuy(pair, price) + nrecent = int(context.args[0]) if context.args else 10 + except (TypeError, ValueError, IndexError): + nrecent = 10 + try: + trades = self._rpc_trade_history( + nrecent + ) + trades_tab = tabulate( + [[arrow.get(trade['open_date']).humanize(), + trade['pair'], + f"{(100 * trade['close_profit']):.2f}% ({trade['close_profit_abs']})"] + for trade in trades['trades']], + headers=[ + 'Open Date', + 'Pair', + f'Profit ({stake_cur})', + ], + tablefmt='simple') + message = (f"{min(trades['trades_count'], nrecent)} recent trades:\n" + + (f"
{trades_tab}
" if trades['trades_count'] > 0 else '')) + self._send_msg(message, parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e)) + + @authorized_only + def _delete_trade(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /delete . + Delete the given trade + :param bot: telegram bot + :param update: message update + :return: None + """ + try: + if not context.args or len(context.args) == 0: + raise RPCException("Trade-id not set.") + trade_id = int(context.args[0]) + msg = self._rpc_delete(trade_id) + self._send_msg(( + '`{result_msg}`\n' + 'Please make sure to take care of this asset on the exchange manually.' + ).format(**msg)) + except RPCException as e: self._send_msg(str(e)) @@ -536,6 +696,26 @@ class Telegram(RPC): except RPCException as e: self._send_msg(str(e)) + @authorized_only + def _locks(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /locks. + Returns the currently active locks + """ + try: + locks = self._rpc_locks() + message = tabulate([[ + lock['pair'], + lock['lock_end_time'], + lock['reason']] for lock in locks['locks']], + headers=['Pair', 'Until', 'Reason'], + tablefmt='simple') + message = "
{}
".format(message) + logger.debug(message) + self._send_msg(message, parse_mode=ParseMode.HTML) + except RPCException as e: + self._send_msg(str(e)) + @authorized_only def _whitelist(self, update: Update, context: CallbackContext) -> None: """ @@ -576,6 +756,38 @@ class Telegram(RPC): except RPCException as e: self._send_msg(str(e)) + @authorized_only + def _logs(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /logs + Shows the latest logs + """ + try: + try: + limit = int(context.args[0]) if context.args else 10 + except (TypeError, ValueError, IndexError): + limit = 10 + logs = RPC._rpc_get_logs(limit)['logs'] + msgs = '' + msg_template = "*{}* {}: {} \\- `{}`" + for logrec in logs: + msg = msg_template.format(escape_markdown(logrec[0], version=2), + escape_markdown(logrec[2], version=2), + escape_markdown(logrec[3], version=2), + escape_markdown(logrec[4], version=2)) + if len(msgs + msg) + 10 >= MAX_TELEGRAM_MESSAGE_LENGTH: + # Send message immediately if it would become too long + self._send_msg(msgs, parse_mode=ParseMode.MARKDOWN_V2) + msgs = msg + '\n' + else: + # Append message to messages to send + msgs += msg + '\n' + + if msgs: + self._send_msg(msgs, parse_mode=ParseMode.MARKDOWN_V2) + except RPCException as e: + self._send_msg(str(e)) + @authorized_only def _edge(self, update: Update, context: CallbackContext) -> None: """ @@ -607,18 +819,23 @@ class Telegram(RPC): " *table :* `will display trades in a table`\n" " `pending buy orders are marked with an asterisk (*)`\n" " `pending sell orders are marked with a double asterisk (**)`\n" + "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n" "*/profit:* `Lists cumulative profit from all finished trades`\n" "*/forcesell |all:* `Instantly sells the given trade or all trades, " "regardless of profit`\n" f"{forcebuy_text if self._config.get('forcebuy_enable', False) else ''}" + "*/delete :* `Instantly delete the given trade in the database`\n" "*/performance:* `Show performance of each finished trade grouped by pair`\n" "*/daily :* `Shows profit or loss per day, over the last n days`\n" - "*/count:* `Show number of trades running compared to allowed number of trades`" - "\n" + "*/stats:* `Shows Wins / losses by Sell reason as well as " + "Avg. holding durationsfor buys and sells.`\n" + "*/count:* `Show number of active trades compared to allowed number of trades`\n" + "*/locks:* `Show currently locked pairs`\n" "*/balance:* `Show account balance per currency`\n" "*/stopbuy:* `Stops buying, but handles open trades gracefully` \n" - "*/reload_conf:* `Reload configuration file` \n" + "*/reload_config:* `Reload configuration file` \n" "*/show_config:* `Show running configuration` \n" + "*/logs [limit]:* `Show latest logs - defaults to 10` \n" "*/whitelist:* `Show current whitelist` \n" "*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs " "to the blacklist.` \n" @@ -648,7 +865,8 @@ class Telegram(RPC): :param update: message update :return: None """ - val = self._rpc_show_config() + val = RPC._rpc_show_config(self._freqtrade.config, self._freqtrade.state) + if val['trailing_stop']: sl_info = ( f"*Initial Stoploss:* `{val['stoploss']}`\n" @@ -669,12 +887,13 @@ class Telegram(RPC): f"*Ask strategy:* ```\n{json.dumps(val['ask_strategy'])}```\n" f"*Bid strategy:* ```\n{json.dumps(val['bid_strategy'])}```\n" f"{sl_info}" - f"*Ticker Interval:* `{val['ticker_interval']}`\n" + f"*Timeframe:* `{val['timeframe']}`\n" f"*Strategy:* `{val['strategy']}`\n" f"*Current state:* `{val['state']}`" ) - def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: + def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, + disable_notification: bool = False) -> None: """ Send given markdown message :param msg: message @@ -682,20 +901,15 @@ class Telegram(RPC): :param parse_mode: telegram parse mode :return: None """ - - keyboard = [['/daily', '/profit', '/balance'], - ['/status', '/status table', '/performance'], - ['/count', '/start', '/stop', '/help']] - - reply_markup = ReplyKeyboardMarkup(keyboard) - + reply_markup = ReplyKeyboardMarkup(self._keyboard) try: try: self._updater.bot.send_message( self._config['telegram']['chat_id'], text=msg, parse_mode=parse_mode, - reply_markup=reply_markup + reply_markup=reply_markup, + disable_notification=disable_notification, ) except NetworkError as network_err: # Sometimes the telegram server resets the current connection, @@ -708,7 +922,8 @@ class Telegram(RPC): self._config['telegram']['chat_id'], text=msg, parse_mode=parse_mode, - reply_markup=reply_markup + reply_markup=reply_markup, + disable_notification=disable_notification, ) except TelegramError as telegram_err: logger.warning( diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 322d990ee..f4008a70f 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -2,9 +2,9 @@ This module manages webhook communication """ import logging -from typing import Any, Dict +from typing import Any, Dict -from requests import post, RequestException +from requests import RequestException, post from freqtrade.rpc import RPC, RPCMessageType @@ -25,7 +25,6 @@ class Webhook(RPC): """ super().__init__(freqtrade) - self._config = freqtrade.config self._url = self._config['webhook']['url'] def cleanup(self) -> None: @@ -48,13 +47,13 @@ class Webhook(RPC): elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION: valuedict = self._config['webhook'].get('webhooksellcancel', None) elif msg['type'] in (RPCMessageType.STATUS_NOTIFICATION, - RPCMessageType.CUSTOM_NOTIFICATION, + RPCMessageType.STARTUP_NOTIFICATION, RPCMessageType.WARNING_NOTIFICATION): valuedict = self._config['webhook'].get('webhookstatus', None) else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) if not valuedict: - logger.info("Message type %s not configured for webhooks", msg['type']) + logger.info("Message type '%s' not configured for webhooks", msg['type']) return payload = {key: value.format(**msg) for (key, value) in valuedict.items()} diff --git a/freqtrade/state.py b/freqtrade/state.py index 38784c6a4..8ddff71d9 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -12,7 +12,7 @@ class State(Enum): """ RUNNING = 1 STOPPED = 2 - RELOAD_CONF = 3 + RELOAD_CONFIG = 3 def __str__(self): return f"{self.name.lower()}" diff --git a/freqtrade/strategy/__init__.py b/freqtrade/strategy/__init__.py index 40a4a0bea..662156ae9 100644 --- a/freqtrade/strategy/__init__.py +++ b/freqtrade/strategy/__init__.py @@ -1 +1,5 @@ -from freqtrade.strategy.interface import IStrategy # noqa: F401 +# flake8: noqa: F401 +from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, + timeframe_to_prev_date, timeframe_to_seconds) +from freqtrade.strategy.interface import IStrategy +from freqtrade.strategy.strategy_helper import merge_informative_pair diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index ed2344a53..027c5d36e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -7,17 +7,18 @@ import warnings from abc import ABC, abstractmethod from datetime import datetime, timezone from enum import Enum -from typing import Dict, NamedTuple, Optional, Tuple +from typing import Dict, List, NamedTuple, Optional, Tuple import arrow from pandas import DataFrame -from freqtrade.data.dataprovider import DataProvider -from freqtrade.exceptions import StrategyError -from freqtrade.exchange import timeframe_to_minutes -from freqtrade.persistence import Trade -from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.data.dataprovider import DataProvider +from freqtrade.exceptions import OperationalException, StrategyError +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.exchange.exchange import timeframe_to_next_date +from freqtrade.persistence import PairLocks, Trade +from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -45,6 +46,10 @@ class SellType(Enum): EMERGENCY_SELL = "emergency_sell" NONE = "" + def __str__(self): + # explicitly convert to String to help with exporting data. + return self.value + class SellCheckTuple(NamedTuple): """ @@ -62,7 +67,7 @@ class IStrategy(ABC): Attributes you can use: minimal_roi -> Dict: Minimal ROI designed for the strategy stoploss -> float: optimal stoploss designed for the strategy - ticker_interval -> str: value of the timeframe (ticker interval) to use with the strategy + timeframe -> str: value of the timeframe (ticker interval) to use with the strategy """ # Strategy interface version # Default to version 2 @@ -85,8 +90,9 @@ class IStrategy(ABC): trailing_stop_positive_offset: float = 0.0 trailing_only_offset_is_reached = False - # associated ticker interval - ticker_interval: str + # associated timeframe + ticker_interval: str # DEPRECATED + timeframe: str # Optional order types order_types: Dict = { @@ -117,6 +123,8 @@ class IStrategy(ABC): # and wallets - access to the current balance. dp: Optional[DataProvider] = None wallets: Optional[Wallets] = None + # container variable for strategy source code + __source__: str = '' # Definition of plot_config. See plotting documentation for more details. plot_config: Dict = {} @@ -125,7 +133,6 @@ class IStrategy(ABC): self.config = config # Dict to determine if analysis is necessary self._last_candle_seen_per_pair: Dict[str, datetime] = {} - self._pair_locked_until: Dict[str, datetime] = {} @abstractmethod def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: @@ -190,6 +197,63 @@ class IStrategy(ABC): """ return False + def bot_loop_start(self, **kwargs) -> None: + """ + Called at the start of the bot iteration (one loop). + Might be used to perform pair-independent tasks + (e.g. gather some remote resource for comparison) + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + """ + pass + + def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, + time_in_force: str, **kwargs) -> bool: + """ + Called right before placing a buy order. + Timing for this function is critical, so avoid doing heavy computations or + network requests in this method. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns True (always confirming). + + :param pair: Pair that's about to be bought. + :param order_type: Order type (as configured in order_types). usually limit or market. + :param amount: Amount in target (quote) currency that's going to be traded. + :param rate: Rate that's going to be used when using limit orders + :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the buy-order is placed on the exchange. + False aborts the process + """ + return True + + def confirm_trade_exit(self, pair: str, trade: Trade, order_type: str, amount: float, + rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool: + """ + Called right before placing a regular sell order. + Timing for this function is critical, so avoid doing heavy computations or + network requests in this method. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns True (always confirming). + + :param pair: Pair that's about to be sold. + :param trade: trade object. + :param order_type: Order type (as configured in order_types). usually limit or market. + :param amount: Amount in quote currency. + :param rate: Rate that's going to be used when using limit orders + :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param sell_reason: Sell reason. + Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', + 'sell_signal', 'force_sell', 'emergency_sell'] + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the sell-order is placed on the exchange. + False aborts the process + """ + return True + def informative_pairs(self) -> ListPairsWithTimeframes: """ Define additional, informative pair/interval combinations to be cached from the exchange. @@ -203,13 +267,17 @@ class IStrategy(ABC): """ return [] +### +# END - Intended to be overridden by strategy +### + def get_strategy_name(self) -> str: """ Returns strategy class name """ return self.__class__.__name__ - def lock_pair(self, pair: str, until: datetime) -> None: + def lock_pair(self, pair: str, until: datetime, reason: str = None) -> None: """ Locks pair until a given timestamp happens. Locked pairs are not analyzed, and are prevented from opening new trades. @@ -218,9 +286,9 @@ class IStrategy(ABC): :param pair: Pair to lock :param until: datetime in UTC until the pair should be blocked from opening new trades. Needs to be timezone aware `datetime.now(timezone.utc)` + :param reason: Optional string explaining why the pair was locked. """ - if pair not in self._pair_locked_until or self._pair_locked_until[pair] < until: - self._pair_locked_until[pair] = until + PairLocks.lock_pair(pair, until, reason) def unlock_pair(self, pair: str) -> None: """ @@ -229,16 +297,25 @@ class IStrategy(ABC): manually from within the strategy, to allow an easy way to unlock pairs. :param pair: Unlock pair to allow trading again """ - if pair in self._pair_locked_until: - del self._pair_locked_until[pair] + PairLocks.unlock_pair(pair, datetime.now(timezone.utc)) - def is_pair_locked(self, pair: str) -> bool: + def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool: """ Checks if a pair is currently locked + The 2nd, optional parameter ensures that locks are applied until the new candle arrives, + and not stop at 14:00:00 - while the next candle arrives at 14:00:02 leaving a gap + of 2 seconds for a buy to happen on an old signal. + :param: pair: "Pair to check" + :param candle_date: Date of the last candle. Optional, defaults to current date + :returns: locking state of the pair in question. """ - if pair not in self._pair_locked_until: - return False - return self._pair_locked_until[pair] >= datetime.now(timezone.utc) + + if not candle_date: + # Simple call ... + return PairLocks.is_pair_locked(pair) + else: + lock_time = timeframe_to_next_date(self.timeframe, candle_date) + return PairLocks.is_pair_locked(pair, lock_time) def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ @@ -272,6 +349,8 @@ class IStrategy(ABC): # Defs that only make change on new candle data. dataframe = self.analyze_ticker(dataframe, metadata) self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] + if self.dp: + self.dp._set_cached_df(pair, self.timeframe, dataframe) else: logger.debug("Skipping TA Analysis for already analyzed candle") dataframe['buy'] = 0 @@ -283,13 +362,53 @@ class IStrategy(ABC): return dataframe + def analyze_pair(self, pair: str) -> None: + """ + Fetch data for this pair from dataprovider and analyze. + Stores the dataframe into the dataprovider. + The analyzed dataframe is then accessible via `dp.get_analyzed_dataframe()`. + :param pair: Pair to analyze. + """ + if not self.dp: + raise OperationalException("DataProvider not found.") + dataframe = self.dp.ohlcv(pair, self.timeframe) + if not isinstance(dataframe, DataFrame) or dataframe.empty: + logger.warning('Empty candle (OHLCV) data for pair %s', pair) + return + + try: + df_len, df_close, df_date = self.preserve_df(dataframe) + + dataframe = strategy_safe_wrapper( + self._analyze_ticker_internal, message="" + )(dataframe, {'pair': pair}) + + self.assert_df(dataframe, df_len, df_close, df_date) + except StrategyError as error: + logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}") + return + + if dataframe.empty: + logger.warning('Empty dataframe for pair %s', pair) + return + + def analyze(self, pairs: List[str]) -> None: + """ + Analyze all pairs using analyze_pair(). + :param pairs: List of pairs to analyze + """ + for pair in pairs: + self.analyze_pair(pair) + @staticmethod def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]: """ keep some data for dataframes """ return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1] def assert_df(self, dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime): - """ make sure data is unmodified """ + """ + Ensure dataframe (length, last candle) was not modified, and has all elements we need. + """ message = "" if df_len != len(dataframe): message = "length" @@ -303,31 +422,17 @@ class IStrategy(ABC): else: raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.") - def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]: + def get_signal(self, pair: str, timeframe: str, dataframe: DataFrame) -> Tuple[bool, bool]: """ - Calculates current signal based several technical analysis indicators + Calculates current signal based based on the buy / sell columns of the dataframe. + Used by Bot to get the signal to buy or sell :param pair: pair in format ANT/BTC - :param interval: Interval to use (in min) - :param dataframe: Dataframe to analyze + :param timeframe: timeframe to use + :param dataframe: Analyzed dataframe to get signal from. :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ if not isinstance(dataframe, DataFrame) or dataframe.empty: - logger.warning('Empty candle (OHLCV) data for pair %s', pair) - return False, False - - try: - df_len, df_close, df_date = self.preserve_df(dataframe) - dataframe = strategy_safe_wrapper( - self._analyze_ticker_internal, message="" - )(dataframe, {'pair': pair}) - self.assert_df(dataframe, df_len, df_close, df_date) - except StrategyError as error: - logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}") - - return False, False - - if dataframe.empty: - logger.warning('Empty dataframe for pair %s', pair) + logger.warning(f'Empty candle (OHLCV) data for pair {pair}') return False, False latest_date = dataframe['date'].max() @@ -336,24 +441,18 @@ class IStrategy(ABC): latest_date = arrow.get(latest_date) # Check if dataframe is out of date - interval_minutes = timeframe_to_minutes(interval) + timeframe_minutes = timeframe_to_minutes(timeframe) offset = self.config.get('exchange', {}).get('outdated_offset', 5) - if latest_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))): + if latest_date < (arrow.utcnow().shift(minutes=-(timeframe_minutes * 2 + offset))): logger.warning( 'Outdated history for pair %s. Last tick is %s minutes old', - pair, - (arrow.utcnow() - latest_date).seconds // 60 + pair, int((arrow.utcnow() - latest_date).total_seconds() // 60) ) return False, False (buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1 - logger.debug( - 'trigger: %s (pair=%s) buy=%s sell=%s', - latest['date'], - pair, - str(buy), - str(sell) - ) + logger.debug('trigger: %s (pair=%s) buy=%s sell=%s', + latest['date'], pair, str(buy), str(sell)) return buy, sell def should_sell(self, trade: Trade, rate: float, date: datetime, buy: bool, @@ -377,40 +476,44 @@ class IStrategy(ABC): current_time=date, current_profit=current_profit, force_stoploss=force_stoploss, high=high) - if stoplossflag.sell_flag: - logger.debug(f"{trade.pair} - Stoploss hit. sell_flag=True, " - f"sell_type={stoplossflag.sell_type}") - return stoplossflag - # Set current rate to high for backtesting sell current_rate = high or rate current_profit = trade.calc_profit_ratio(current_rate) config_ask_strategy = self.config.get('ask_strategy', {}) - if buy and config_ask_strategy.get('ignore_roi_if_buy_signal', False): - # This one is noisy, commented out - # logger.debug(f"{trade.pair} - Buy signal still active. sell_flag=False") - return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) + # if buy signal and ignore_roi is set, we don't need to evaluate min_roi. + roi_reached = (not (buy and config_ask_strategy.get('ignore_roi_if_buy_signal', False)) + and self.min_roi_reached(trade=trade, current_profit=current_profit, + current_time=date)) - # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) - if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): + if config_ask_strategy.get('sell_profit_only', False) and trade.calc_profit(rate=rate) <= 0: + # Negative profits and sell_profit_only - ignore sell signal + sell_signal = False + else: + sell_signal = sell and not buy and config_ask_strategy.get('use_sell_signal', True) + # TODO: return here if sell-signal should be favored over ROI + + # Start evaluations + # Sequence: + # ROI (if not stoploss) + # Sell-signal + # Stoploss + if roi_reached and stoplossflag.sell_type != SellType.STOP_LOSS: logger.debug(f"{trade.pair} - Required profit reached. sell_flag=True, " f"sell_type=SellType.ROI") return SellCheckTuple(sell_flag=True, sell_type=SellType.ROI) - if config_ask_strategy.get('sell_profit_only', False): - # This one is noisy, commented out - # logger.debug(f"{trade.pair} - Checking if trade is profitable...") - if trade.calc_profit(rate=rate) <= 0: - # This one is noisy, commented out - # logger.debug(f"{trade.pair} - Trade is not profitable. sell_flag=False") - return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) - - if sell and not buy and config_ask_strategy.get('use_sell_signal', True): + if sell_signal: logger.debug(f"{trade.pair} - Sell signal received. sell_flag=True, " f"sell_type=SellType.SELL_SIGNAL") return SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL) + if stoplossflag.sell_flag: + + logger.debug(f"{trade.pair} - Stoploss hit. sell_flag=True, " + f"sell_type={stoplossflag.sell_type}") + return stoplossflag + # This one is noisy, commented out... # logger.debug(f"{trade.pair} - No sell signal. sell_flag=False") return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) @@ -448,8 +551,7 @@ class IStrategy(ABC): # evaluate if the stoploss was hit if stoploss is not on exchange # in Dry-Run, this handles stoploss logic as well, as the logic will not be different to # regular stoploss handling. - if ((self.stoploss is not None) and - (trade.stop_loss >= current_rate) and + if ((trade.stop_loss >= current_rate) and (not self.order_types.get('stoploss_on_exchange') or self.config['dry_run'])): sell_type = SellType.STOP_LOSS @@ -499,7 +601,8 @@ class IStrategy(ABC): def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]: """ - Creates a dataframe and populates indicators for given candle (OHLCV) data + Populates indicators for given candle (OHLCV) data (for multiple pairs) + Does not run advice_buy or advise_sell! Used by optimize operations only, not during dry / live runs. Using .copy() to get a fresh copy of the dataframe for every strategy run. Has positive effects on memory usage for whatever reason - also when diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py new file mode 100644 index 000000000..ea0e234ec --- /dev/null +++ b/freqtrade/strategy/strategy_helper.py @@ -0,0 +1,49 @@ +import pandas as pd + +from freqtrade.exchange import timeframe_to_minutes + + +def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, + timeframe: str, timeframe_inf: str, ffill: bool = True) -> pd.DataFrame: + """ + Correctly merge informative samples to the original dataframe, avoiding lookahead bias. + + Since dates are candle open dates, merging a 15m candle that starts at 15:00, and a + 1h candle that starts at 15:00 will result in all candles to know the close at 16:00 + which they should not know. + + Moves the date of the informative pair by 1 time interval forward. + This way, the 14:00 1h candle is merged to 15:00 15m candle, since the 14:00 1h candle is the + last candle that's closed at 15:00, 15:15, 15:30 or 15:45. + + Assuming inf_tf = '1d' - then the resulting columns will be: + date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d + + :param dataframe: Original dataframe + :param informative: Informative pair, most likely loaded via dp.get_pair_dataframe + :param timeframe: Timeframe of the original pair sample. + :param timeframe_inf: Timeframe of the informative pair sample. + :param ffill: Forwardfill missing values - optional but usually required + """ + + minutes_inf = timeframe_to_minutes(timeframe_inf) + minutes = timeframe_to_minutes(timeframe) + if minutes >= minutes_inf: + # No need to forwardshift if the timeframes are identical + informative['date_merge'] = informative["date"] + else: + informative['date_merge'] = informative["date"] + pd.to_timedelta(minutes_inf, 'm') + + # Rename columns to be unique + informative.columns = [f"{col}_{timeframe_inf}" for col in informative.columns] + + # Combine the 2 dataframes + # all indicators on the informative sample MUST be calculated before this point + dataframe = pd.merge(dataframe, informative, left_on='date', + right_on=f'date_merge_{timeframe_inf}', how='left') + dataframe = dataframe.drop(f'date_merge_{timeframe_inf}', axis=1) + + if ffill: + dataframe = dataframe.ffill() + + return dataframe diff --git a/freqtrade/strategy/strategy_wrapper.py b/freqtrade/strategy/strategy_wrapper.py index 7b9da9140..121189b68 100644 --- a/freqtrade/strategy/strategy_wrapper.py +++ b/freqtrade/strategy/strategy_wrapper.py @@ -2,10 +2,11 @@ import logging from freqtrade.exceptions import StrategyError + logger = logging.getLogger(__name__) -def strategy_safe_wrapper(f, message: str = "", default_retval=None): +def strategy_safe_wrapper(f, message: str = "", default_retval=None, supress_error=False): """ Wrapper around user-provided methods and functions. Caches all exceptions and returns either the default_retval (if it's not None) or raises @@ -20,7 +21,7 @@ def strategy_safe_wrapper(f, message: str = "", default_retval=None): f"Strategy caused the following exception: {error}" f"{f}" ) - if default_retval is None: + if default_retval is None and not supress_error: raise StrategyError(str(error)) from error return default_retval except Exception as error: @@ -28,7 +29,7 @@ def strategy_safe_wrapper(f, message: str = "", default_retval=None): f"{message}" f"Unexpected error {error} calling {f}" ) - if default_retval is None: + if default_retval is None and not supress_error: raise StrategyError(str(error)) from error return default_retval diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index 5339595e8..b362690f9 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -4,7 +4,7 @@ "stake_amount": {{ stake_amount }}, "tradable_balance_ratio": 0.99, "fiat_display_currency": "{{ fiat_display_currency }}", - "ticker_interval": "{{ ticker_interval }}", + "timeframe": "{{ timeframe }}", "dry_run": {{ dry_run | lower }}, "cancel_open_orders_on_exit": false, "unfilledtimeout": { @@ -59,6 +59,7 @@ "listen_port": 8080, "verbosity": "info", "jwt_secret_key": "somethingrandom", + "CORS_origins": [], "username": "", "password": "" }, diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index c37164568..4a1b43e36 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -51,8 +51,8 @@ class {{ strategy }}(IStrategy): # trailing_stop_positive = 0.01 # trailing_stop_positive_offset = 0.0 # Disabled / not configured - # Optimal ticker interval for the strategy. - ticker_interval = '5m' + # Optimal timeframe for the strategy. + timeframe = '5m' # Run "populate_indicators()" only for new candle. process_only_new_candles = False @@ -63,7 +63,7 @@ class {{ strategy }}(IStrategy): ignore_roi_if_buy_signal = False # Number of candles the strategy requires before producing valid signals - startup_candle_count: int = 20 + startup_candle_count: int = 30 # Optional order type mapping. order_types = { diff --git a/freqtrade/templates/sample_hyperopt.py b/freqtrade/templates/sample_hyperopt.py index 0b6d030db..10743e911 100644 --- a/freqtrade/templates/sample_hyperopt.py +++ b/freqtrade/templates/sample_hyperopt.py @@ -1,4 +1,5 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement +# isort: skip_file # --- Do not remove these libs --- from functools import reduce diff --git a/freqtrade/templates/sample_hyperopt_advanced.py b/freqtrade/templates/sample_hyperopt_advanced.py index 7f05c4430..52e397466 100644 --- a/freqtrade/templates/sample_hyperopt_advanced.py +++ b/freqtrade/templates/sample_hyperopt_advanced.py @@ -1,5 +1,5 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - +# isort: skip_file # --- Do not remove these libs --- from functools import reduce from typing import Any, Callable, Dict, List diff --git a/freqtrade/templates/sample_hyperopt_loss.py b/freqtrade/templates/sample_hyperopt_loss.py index 4173d97f5..59e6d814a 100644 --- a/freqtrade/templates/sample_hyperopt_loss.py +++ b/freqtrade/templates/sample_hyperopt_loss.py @@ -1,10 +1,11 @@ -from math import exp from datetime import datetime +from math import exp from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss + # Define some constants: # set TARGET_TRADES to suit your number concurrent trades so its realistic diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index f78489173..b3f9fef07 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -1,5 +1,5 @@ # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - +# isort: skip_file # --- Do not remove these libs --- import numpy as np # noqa import pandas as pd # noqa @@ -53,7 +53,7 @@ class SampleStrategy(IStrategy): # trailing_stop_positive_offset = 0.0 # Disabled / not configured # Optimal ticker interval for the strategy. - ticker_interval = '5m' + timeframe = '5m' # Run "populate_indicators()" only for new candle. process_only_new_candles = False @@ -64,7 +64,7 @@ class SampleStrategy(IStrategy): ignore_roi_if_buy_signal = False # Number of candles the strategy requires before producing valid signals - startup_candle_count: int = 20 + startup_candle_count: int = 30 # Optional order type mapping. order_types = { @@ -184,6 +184,8 @@ class SampleStrategy(IStrategy): dataframe['fastk'] = stoch_fast['fastk'] # # Stochastic RSI + # Please read https://github.com/freqtrade/freqtrade/issues/2961 before using this. + # STOCHRSI is NOT aligned with tradingview, which may result in non-expected results. # stoch_rsi = ta.STOCHRSI(dataframe) # dataframe['fastd_rsi'] = stoch_rsi['fastd'] # dataframe['fastk_rsi'] = stoch_rsi['fastk'] diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index dffa308ce..c6e64c74e 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -34,7 +34,7 @@ "# config = Configuration.from_files([\"config.json\"])\n", "\n", "# Define some constants\n", - "config[\"ticker_interval\"] = \"5m\"\n", + "config[\"timeframe\"] = \"5m\"\n", "# Name of the strategy class\n", "config[\"strategy\"] = \"SampleStrategy\"\n", "# Location of the data\n", @@ -53,7 +53,7 @@ "from freqtrade.data.history import load_pair_history\n", "\n", "candles = load_pair_history(datadir=data_location,\n", - " timeframe=config[\"ticker_interval\"],\n", + " timeframe=config[\"timeframe\"],\n", " pair=pair)\n", "\n", "# Confirm success\n", @@ -136,10 +136,51 @@ "metadata": {}, "outputs": [], "source": [ - "from freqtrade.data.btanalysis import load_backtest_data\n", + "from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats\n", "\n", - "# Load backtest results\n", - "trades = load_backtest_data(config[\"user_data_dir\"] / \"backtest_results/backtest-result.json\")\n", + "# if backtest_dir points to a directory, it'll automatically load the last backtest file.\n", + "backtest_dir = config[\"user_data_dir\"] / \"backtest_results\"\n", + "# backtest_dir can also point to a specific file \n", + "# backtest_dir = config[\"user_data_dir\"] / \"backtest_results/backtest-result-2020-07-01_20-04-22.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# You can get the full backtest statistics by using the following command.\n", + "# This contains all information used to generate the backtest result.\n", + "stats = load_backtest_stats(backtest_dir)\n", + "\n", + "strategy = 'SampleStrategy'\n", + "# All statistics are available per strategy, so if `--strategy-list` was used during backtest, this will be reflected here as well.\n", + "# Example usages:\n", + "print(stats['strategy'][strategy]['results_per_pair'])\n", + "# Get pairlist used for this backtest\n", + "print(stats['strategy'][strategy]['pairlist'])\n", + "# Get market change (average change of all pairs from start to end of the backtest period)\n", + "print(stats['strategy'][strategy]['market_change'])\n", + "# Maximum drawdown ()\n", + "print(stats['strategy'][strategy]['max_drawdown'])\n", + "# Maximum drawdown start and end\n", + "print(stats['strategy'][strategy]['drawdown_start'])\n", + "print(stats['strategy'][strategy]['drawdown_end'])\n", + "\n", + "\n", + "# Get strategy comparison (only relevant if multiple strategies were compared)\n", + "print(stats['strategy_comparison'])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load backtested trades as dataframe\n", + "trades = load_backtest_data(backtest_dir)\n", "\n", "# Show value-counts per pair\n", "trades.groupby(\"pair\")[\"sell_reason\"].value_counts()" diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 60a358bec..57d2ca665 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -62,6 +62,8 @@ dataframe['fastd'] = stoch_fast['fastd'] dataframe['fastk'] = stoch_fast['fastk'] # # Stochastic RSI +# Please read https://github.com/freqtrade/freqtrade/issues/2961 before using this. +# STOCHRSI is NOT aligned with tradingview, which may result in non-expected results. # stoch_rsi = ta.STOCHRSI(dataframe) # dataframe['fastd_rsi'] = stoch_rsi['fastd'] # dataframe['fastk_rsi'] = stoch_rsi['fastk'] diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 0ca35e117..5ca6e6971 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -1,4 +1,65 @@ +def bot_loop_start(self, **kwargs) -> None: + """ + Called at the start of the bot iteration (one loop). + Might be used to perform pair-independent tasks + (e.g. gather some remote ressource for comparison) + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, this simply does nothing. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + """ + pass + +def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, + time_in_force: str, **kwargs) -> bool: + """ + Called right before placing a buy order. + Timing for this function is critical, so avoid doing heavy computations or + network requests in this method. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns True (always confirming). + + :param pair: Pair that's about to be bought. + :param order_type: Order type (as configured in order_types). usually limit or market. + :param amount: Amount in target (quote) currency that's going to be traded. + :param rate: Rate that's going to be used when using limit orders + :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the buy-order is placed on the exchange. + False aborts the process + """ + return True + +def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float, + rate: float, time_in_force: str, sell_reason: str, **kwargs) -> bool: + """ + Called right before placing a regular sell order. + Timing for this function is critical, so avoid doing heavy computations or + network requests in this method. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, returns True (always confirming). + + :param pair: Pair that's about to be sold. + :param trade: trade object. + :param order_type: Order type (as configured in order_types). usually limit or market. + :param amount: Amount in quote currency. + :param rate: Rate that's going to be used when using limit orders + :param time_in_force: Time in force. Defaults to GTC (Good-til-cancelled). + :param sell_reason: Sell reason. + Can be any of ['roi', 'stop_loss', 'stoploss_on_exchange', 'trailing_stop_loss', + 'sell_signal', 'force_sell', 'emergency_sell'] + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the sell-order is placed on the exchange. + False aborts the process + """ + return True + def check_buy_timeout(self, pair: str, trade: 'Trade', order: dict, **kwargs) -> bool: """ Check buy timeout function callback. diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index bef140396..4c0fb5b5c 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -19,14 +19,15 @@ # limitations under the License. # -import warnings import sys +import warnings from datetime import datetime, timedelta import numpy as np import pandas as pd from pandas.core.base import PandasObject + # ============================================= # check min, python version if sys.version_info < (3, 4): @@ -222,7 +223,7 @@ def crossed(series1, series2, direction=None): if isinstance(series1, np.ndarray): series1 = pd.Series(series1) - if isinstance(series2, (float, int, np.ndarray)): + if isinstance(series2, (float, int, np.ndarray, np.integer, np.floating)): series2 = pd.Series(index=series1.index, data=series2) if direction is None or direction == "above": diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index b913155bc..3680dd416 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -2,6 +2,7 @@ """ Wallet """ import logging +from copy import deepcopy from typing import Any, Dict, NamedTuple import arrow @@ -9,6 +10,7 @@ import arrow from freqtrade.exchange import Exchange from freqtrade.persistence import Trade + logger = logging.getLogger(__name__) @@ -93,6 +95,10 @@ class Wallets: balances[currency].get('used', None), balances[currency].get('total', None) ) + # Remove currencies no longer in get_balances output + for currency in deepcopy(self._wallets): + if currency not in balances: + del self._wallets[currency] def update(self, require_update: bool = True) -> None: """ @@ -102,13 +108,13 @@ class Wallets: for trading operations, the latest balance is needed. :param require_update: Allow skipping an update if balances were recently refreshed """ - if (require_update or (self._last_wallet_refresh + 3600 < arrow.utcnow().timestamp)): + if (require_update or (self._last_wallet_refresh + 3600 < arrow.utcnow().int_timestamp)): if self._config['dry_run']: self._update_dry() else: self._update_live() logger.info('Wallets synced.') - self._last_wallet_refresh = arrow.utcnow().timestamp + self._last_wallet_refresh = arrow.utcnow().int_timestamp def get_all_balances(self) -> Dict[str, Any]: return self._wallets diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 3f5ab734e..ec9331eef 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -15,6 +15,7 @@ from freqtrade.exceptions import OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot from freqtrade.state import State + logger = logging.getLogger(__name__) @@ -71,7 +72,7 @@ class Worker: state = None while True: state = self._worker(old_state=state) - if state == State.RELOAD_CONF: + if state == State.RELOAD_CONFIG: self._reconfigure() def _worker(self, old_state: Optional[State]) -> State: @@ -90,6 +91,9 @@ class Worker: if state == State.RUNNING: self.freqtrade.startup() + if state == State.STOPPED: + self.freqtrade.check_for_open_trades() + # Reset heartbeat timestamp to log the heartbeat message at # first throttling iteration when the state changes self._heartbeat_msg = 0 diff --git a/mkdocs.yml b/mkdocs.yml index ae24e150c..a7ae0cc96 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,8 +1,12 @@ site_name: Freqtrade nav: - Home: index.md - - Installation Docker: docker.md - - Installation: installation.md + - Quickstart with Docker: docker_quickstart.md + - Installation: + - Docker without docker-compose: docker.md + - Linux/MacOS/Raspberry: installation.md + - Windows: windows_installation.md + - Freqtrade Basics: bot-basics.md - Configuration: configuration.md - Strategy Customization: strategy-customization.md - Stoploss: stoploss.md @@ -15,18 +19,20 @@ nav: - Backtesting: backtesting.md - Hyperopt: hyperopt.md - Edge Positioning: edge.md + - Plugins: plugins.md - Utility Subcommands: utils.md - - Exchange-specific Notes: exchanges.md - FAQ: faq.md - Data Analysis: - Jupyter Notebooks: data-analysis.md - Strategy analysis: strategy_analysis_example.md - Plotting: plotting.md - SQL Cheatsheet: sql_cheatsheet.md + - Exchange-specific Notes: exchanges.md - Advanced Post-installation Tasks: advanced-setup.md - Advanced Strategy: strategy-advanced.md - Advanced Hyperopt: advanced-hyperopt.md - Sandbox Testing: sandbox-testing.md + - Updating Freqtrade: updating.md - Deprecated Features: deprecated.md - Contributors Guide: developer.md theme: @@ -38,22 +44,29 @@ theme: accent: 'tear' extra_css: - 'stylesheets/ft.extra.css' +extra_javascript: + - javascripts/config.js + - https://polyfill.io/v3/polyfill.min.js?features=es6 + - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js markdown_extensions: - admonition + - footnotes - codehilite: guess_lang: false - toc: permalink: true - - pymdownx.arithmatex - - pymdownx.caret - - pymdownx.critic + - pymdownx.arithmatex: + generic: true - pymdownx.details - pymdownx.inlinehilite - pymdownx.magiclink - - pymdownx.mark + - pymdownx.pathconverter - pymdownx.smartsymbols + - pymdownx.snippets: + base_path: docs + check_paths: true + - pymdownx.tabbed - pymdownx.superfences - pymdownx.tasklist: custom_checkbox: true - - pymdownx.tilde - mdx_truly_sane_lists diff --git a/requirements-common.txt b/requirements-common.txt deleted file mode 100644 index dab3a5da4..000000000 --- a/requirements-common.txt +++ /dev/null @@ -1,35 +0,0 @@ -# requirements without requirements installable via conda -# mainly used for Raspberry pi installs -ccxt==1.29.52 -SQLAlchemy==1.3.17 -python-telegram-bot==12.7 -arrow==0.15.6 -cachetools==4.1.0 -requests==2.23.0 -urllib3==1.25.9 -wrapt==1.12.1 -jsonschema==3.2.0 -TA-Lib==0.4.18 -tabulate==0.8.7 -pycoingecko==1.2.0 -jinja2==2.11.2 - -# find first, C search in arrays -py_find_1st==1.1.4 - -# Load ticker files 30% faster -python-rapidjson==0.9.1 - -# Notify systemd -sdnotify==0.3.2 - -# Api server -flask==1.1.2 -flask-jwt-extended==3.24.1 -flask-cors==3.0.8 - -# Support for colorized terminal output -colorama==0.4.3 -# Building config files interactively -questionary==1.5.2 -prompt-toolkit==3.0.5 diff --git a/requirements-dev.txt b/requirements-dev.txt index d62b768c1..a2da87430 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,16 +3,17 @@ -r requirements-plot.txt -r requirements-hyperopt.txt -coveralls==2.0.0 -flake8==3.8.2 +coveralls==2.2.0 +flake8==3.8.4 flake8-type-annotations==0.1.0 -flake8-tidy-imports==4.1.0 -mypy==0.780 -pytest==5.4.3 -pytest-asyncio==0.12.0 -pytest-cov==2.9.0 -pytest-mock==3.1.1 +flake8-tidy-imports==4.2.1 +mypy==0.790 +pytest==6.2.1 +pytest-asyncio==0.14.0 +pytest-cov==2.10.1 +pytest-mock==3.4.0 pytest-random-order==1.0.4 +isort==5.6.4 # Convert jupyter notebooks to markdown documents -nbconvert==5.6.1 +nbconvert==6.0.7 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index e1b3fef4f..c51062bf7 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,9 +2,9 @@ -r requirements.txt # Required for hyperopt -scipy==1.4.1 -scikit-learn==0.23.1 -scikit-optimize==0.7.4 +scipy==1.5.4 +scikit-learn==0.23.2 +scikit-optimize==0.8.1 filelock==3.0.12 -joblib==0.15.1 -progressbar2==3.51.3 +joblib==1.0.0 +progressbar2==3.53.1 diff --git a/requirements-plot.txt b/requirements-plot.txt index cb13a59bf..3e31a24ae 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.8.1 +plotly==4.14.1 diff --git a/requirements.txt b/requirements.txt index 2c68b8f2c..2c565fee5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,39 @@ -# Load common requirements --r requirements-common.txt +numpy==1.19.4 +pandas==1.1.5 -numpy==1.18.5 -pandas==1.0.4 +ccxt==1.39.52 +aiohttp==3.7.3 +SQLAlchemy==1.3.22 +python-telegram-bot==13.1 +arrow==0.17.0 +cachetools==4.2.0 +requests==2.25.1 +urllib3==1.26.2 +wrapt==1.12.1 +jsonschema==3.2.0 +TA-Lib==0.4.19 +tabulate==0.8.7 +pycoingecko==1.4.0 +jinja2==2.11.2 +tables==3.6.1 +blosc==1.9.2 + +# find first, C search in arrays +py_find_1st==1.1.4 + +# Load ticker files 30% faster +python-rapidjson==1.0 + +# Notify systemd +sdnotify==0.3.2 + +# Api server +flask==1.1.2 +flask-jwt-extended==3.25.0 +flask-cors==3.0.9 + +# Support for colorized terminal output +colorama==0.4.4 +# Building config files interactively +questionary==1.9.0 +prompt-toolkit==3.0.8 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index b26c32479..2232b8421 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -10,8 +10,8 @@ so it can be used as a standalone script. import argparse import inspect import json -import re import logging +import re import sys from pathlib import Path from urllib.parse import urlencode, urlparse, urlunparse @@ -20,6 +20,7 @@ import rapidjson import requests from requests.exceptions import ConnectionError + logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', @@ -62,6 +63,9 @@ class FtRestClient(): def _get(self, apipath, params: dict = None): return self._call("GET", apipath, params=params) + def _delete(self, apipath, params: dict = None): + return self._call("DELETE", apipath, params=params) + def _post(self, apipath, params: dict = None, data: dict = None): return self._call("POST", apipath, params=params, data=data) @@ -80,18 +84,18 @@ class FtRestClient(): return self._post("stop") def stopbuy(self): - """Stop buying (but handle sells gracefully). Use `reload_conf` to reset. + """Stop buying (but handle sells gracefully). Use `reload_config` to reset. :return: json object """ return self._post("stopbuy") - def reload_conf(self): + def reload_config(self): """Reload configuration. :return: json object """ - return self._post("reload_conf") + return self._post("reload_config") def balance(self): """Get the account balance. @@ -107,6 +111,13 @@ class FtRestClient(): """ return self._get("count") + def locks(self): + """Return current locks + + :return: json object + """ + return self._get("locks") + def daily(self, days=None): """Return the amount of open trades. @@ -128,6 +139,13 @@ class FtRestClient(): """ return self._get("profit") + def stats(self): + """Return the stats report (durations, sell-reasons). + + :return: json object + """ + return self._get("stats") + def performance(self): """Return the performance of the different coins. @@ -156,6 +174,14 @@ class FtRestClient(): """ return self._get("show_config") + def logs(self, limit=None): + """Show latest logs. + + :param limit: Limits log messages to the last logs. No limit to get all the trades. + :return: json object + """ + return self._get("logs", params={"limit": limit} if limit else 0) + def trades(self, limit=None): """Return trades history. @@ -164,6 +190,15 @@ class FtRestClient(): """ return self._get("trades", params={"limit": limit} if limit else 0) + def delete_trade(self, trade_id): + """Delete trade from the database. + Tries to close open orders. Requires manual handling of this asset on the exchange. + + :param trade_id: Deletes the trade with this ID from the database. + :return: json object + """ + return self._delete("trades/{}".format(trade_id)) + def whitelist(self): """Show the current whitelist. @@ -203,6 +238,70 @@ class FtRestClient(): return self._post("forcesell", data={"tradeid": tradeid}) + def strategies(self): + """Lists available strategies + + :return: json object + """ + return self._get("strategies") + + def strategy(self, strategy): + """Get strategy details + + :param strategy: Strategy class name + :return: json object + """ + return self._get(f"strategy/{strategy}") + + def plot_config(self): + """Return plot configuration if the strategy defines one. + + :return: json object + """ + return self._get("plot_config") + + def available_pairs(self, timeframe=None, stake_currency=None): + """Return available pair (backtest data) based on timeframe / stake_currency selection + + :param timeframe: Only pairs with this timeframe available. + :param stake_currency: Only pairs that include this timeframe + :return: json object + """ + return self._get("available_pairs", params={ + "stake_currency": stake_currency if timeframe else '', + "timeframe": timeframe if timeframe else '', + }) + + def pair_candles(self, pair, timeframe, limit=None): + """Return live dataframe for . + + :param pair: Pair to get data for + :param timeframe: Only pairs with this timeframe available. + :param limit: Limit result to the last n candles. + :return: json object + """ + return self._get("available_pairs", params={ + "pair": pair, + "timeframe": timeframe, + "limit": limit, + }) + + def pair_history(self, pair, timeframe, strategy, timerange=None): + """Return historic, analyzed dataframe + + :param pair: Pair to get data for + :param timeframe: Only pairs with this timeframe available. + :param strategy: Strategy to analyze and get values for + :param timerange: Timerange to get data for (same format than --timerange endpoints) + :return: json object + """ + return self._get("pair_history", params={ + "pair": pair, + "timeframe": timeframe, + "strategy": strategy, + "timerange": timerange if timerange else '', + }) + def add_arguments(): parser = argparse.ArgumentParser() @@ -264,11 +363,11 @@ def main(args): print_commands() sys.exit() - config = load_config(args["config"]) - url = config.get("api_server", {}).get("server_url", "127.0.0.1") - port = config.get("api_server", {}).get("listen_port", "8080") - username = config.get("api_server", {}).get("username") - password = config.get("api_server", {}).get("password") + config = load_config(args['config']) + url = config.get('api_server', {}).get('server_url', '127.0.0.1') + port = config.get('api_server', {}).get('listen_port', '8080') + username = config.get('api_server', {}).get('username') + password = config.get('api_server', {}).get('password') server_url = f"http://{url}:{port}" client = FtRestClient(server_url, username, password) diff --git a/setup.cfg b/setup.cfg index 34f25482b..be2cd450c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,6 +8,11 @@ exclude = .eggs, user_data, +[isort] +line_length=100 +multi_line_output=0 +lines_after_imports=2 + [mypy] ignore_missing_imports = True diff --git a/setup.py b/setup.py index 20963a15f..030980c96 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,15 @@ from sys import version_info + from setuptools import setup -if version_info.major == 3 and version_info.minor < 6 or \ + +if version_info.major == 3 and version_info.minor < 7 or \ version_info.major < 3: - print('Your Python interpreter must be 3.6 or greater!') + print('Your Python interpreter must be 3.7 or greater!') exit(1) from pathlib import Path # noqa: E402 + from freqtrade import __version__ # noqa: E402 @@ -62,11 +65,11 @@ setup(name='freqtrade', setup_requires=['pytest-runner', 'numpy'], tests_require=['pytest', 'pytest-asyncio', 'pytest-cov', 'pytest-mock', ], install_requires=[ - # from requirements-common.txt - 'ccxt>=1.18.1080', + # from requirements.txt + 'ccxt>=1.24.96', 'SQLAlchemy', 'python-telegram-bot', - 'arrow', + 'arrow>=0.17.0', 'cachetools', 'requests', 'urllib3', @@ -82,9 +85,10 @@ setup(name='freqtrade', 'jinja2', 'questionary', 'prompt-toolkit', - # from requirements.txt 'numpy', 'pandas', + 'tables', + 'blosc', ], extras_require={ 'api': api, @@ -105,8 +109,8 @@ setup(name='freqtrade', 'Environment :: Console', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Operating System :: MacOS', 'Operating System :: Unix', 'Topic :: Office/Business :: Financial :: Investment', diff --git a/setup.sh b/setup.sh index 918c41e6b..b270146c1 100755 --- a/setup.sh +++ b/setup.sh @@ -25,6 +25,14 @@ function check_installed_python() { return fi + which python3.9 + if [ $? -eq 0 ]; then + echo "using Python 3.9" + PYTHON=python3.9 + check_installed_pip + return + fi + which python3.7 if [ $? -eq 0 ]; then echo "using Python 3.7" @@ -33,16 +41,9 @@ function check_installed_python() { return fi - which python3.6 - if [ $? -eq 0 ]; then - echo "using Python 3.6" - PYTHON=python3.6 - check_installed_pip - return - fi if [ -z ${PYTHON} ]; then - echo "No usable python found. Please make sure to have python3.6 or python3.7 installed" + echo "No usable python found. Please make sure to have python3.7 or newer installed" exit 1 fi } @@ -56,18 +57,45 @@ function updateenv() { exit 1 fi source .env/bin/activate + SYS_ARCH=$(uname -m) echo "pip install in-progress. Please wait..." ${PYTHON} -m pip install --upgrade pip read -p "Do you want to install dependencies for dev [y/N]? " if [[ $REPLY =~ ^[Yy]$ ]] then - ${PYTHON} -m pip install --upgrade -r requirements-dev.txt + REQUIREMENTS=requirements-dev.txt else - ${PYTHON} -m pip install --upgrade -r requirements.txt - echo "Dev dependencies ignored." + REQUIREMENTS=requirements.txt + fi + REQUIREMENTS_HYPEROPT="" + REQUIREMENTS_PLOT="" + read -p "Do you want to install plotting dependencies (plotly) [y/N]? " + if [[ $REPLY =~ ^[Yy]$ ]] + then + REQUIREMENTS_PLOT="-r requirements-plot.txt" + fi + if [ "${SYS_ARCH}" == "armv7l" ]; then + echo "Detected Raspberry, installing cython, skipping hyperopt installation." + ${PYTHON} -m pip install --upgrade cython + else + # Is not Raspberry + read -p "Do you want to install hyperopt dependencies [y/N]? " + if [[ $REPLY =~ ^[Yy]$ ]] + then + REQUIREMENTS_HYPEROPT="-r requirements-hyperopt.txt" + fi fi + ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} + if [ $? -ne 0 ]; then + echo "Failed installing dependencies" + exit 1 + fi ${PYTHON} -m pip install -e . + if [ $? -ne 0 ]; then + echo "Failed installing Freqtrade" + exit 1 + fi echo "pip install completed" echo } @@ -120,13 +148,13 @@ function update() { updateenv } -# Reset Develop or Master branch +# Reset Develop or Stable branch function reset() { echo "----------------------------" echo "Reseting branch and virtual env" echo "----------------------------" - if [ "1" == $(git branch -vv |grep -cE "\* develop|\* master") ] + if [ "1" == $(git branch -vv |grep -cE "\* develop|\* stable") ] then read -p "Reset git branch? (This will remove all changes you made!) [y/N]? " @@ -134,22 +162,22 @@ function reset() { git fetch -a - if [ "1" == $(git branch -vv |grep -c "* develop") ] + if [ "1" == $(git branch -vv | grep -c "* develop") ] then echo "- Hard resetting of 'develop' branch." git reset --hard origin/develop - elif [ "1" == $(git branch -vv |grep -c "* master") ] + elif [ "1" == $(git branch -vv | grep -c "* stable") ] then - echo "- Hard resetting of 'master' branch." - git reset --hard origin/master + echo "- Hard resetting of 'stable' branch." + git reset --hard origin/stable fi fi else - echo "Reset ignored because you are not on 'master' or 'develop'." + echo "Reset ignored because you are not on 'stable' or 'develop'." fi if [ -d ".env" ]; then - echo "- Delete your previous virtual env" + echo "- Deleting your previous virtual env" rm -rf .env fi echo @@ -253,7 +281,7 @@ function install() { echo "Run the bot !" echo "-------------------------" echo "You can now use the bot by executing 'source .env/bin/activate; freqtrade '." - echo "You can see the list of available bot subcommands by executing 'source .env/bin/activate; freqtrade --help'." + echo "You can see the list of available bot sub-commands by executing 'source .env/bin/activate; freqtrade --help'." echo "You verify that freqtrade is installed successfully by running 'source .env/bin/activate; freqtrade --version'." } @@ -270,12 +298,12 @@ function help() { echo "usage:" echo " -i,--install Install freqtrade from scratch" echo " -u,--update Command git pull to update." - echo " -r,--reset Hard reset your develop/master branch." + echo " -r,--reset Hard reset your develop/stable branch." echo " -c,--config Easy config generator (Will override your existing file)." echo " -p,--plot Install dependencies for Plotting scripts." } -# Verify if 3.6 or 3.7 is installed +# Verify if 3.7 or 3.8 is installed check_installed_python case $* in diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py index d4ebe1de2..291720f4b 100644 --- a/tests/commands/test_build_config.py +++ b/tests/commands/test_build_config.py @@ -4,10 +4,8 @@ from unittest.mock import MagicMock import pytest import rapidjson -from freqtrade.commands.build_config_commands import (ask_user_config, - ask_user_overwrite, - start_new_config, - validate_is_float, +from freqtrade.commands.build_config_commands import (ask_user_config, ask_user_overwrite, + start_new_config, validate_is_float, validate_is_int) from freqtrade.exceptions import OperationalException from tests.conftest import get_args, log_has_re @@ -44,7 +42,7 @@ def test_start_new_config(mocker, caplog, exchange): 'stake_currency': 'USDT', 'stake_amount': 100, 'fiat_display_currency': 'EUR', - 'ticker_interval': '15m', + 'timeframe': '15m', 'dry_run': True, 'exchange_name': exchange, 'exchange_key': 'sampleKey', @@ -68,7 +66,7 @@ def test_start_new_config(mocker, caplog, exchange): result = rapidjson.loads(wt_mock.call_args_list[0][0][0], parse_mode=rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS) assert result['exchange']['name'] == exchange - assert result['ticker_interval'] == '15m' + assert result['timeframe'] == '15m' def test_start_new_config_exists(mocker, caplog): diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 46350beff..26e0c4a79 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -2,22 +2,21 @@ import re from pathlib import Path from unittest.mock import MagicMock, PropertyMock +import arrow import pytest -from freqtrade.commands import (start_convert_data, start_create_userdir, - start_download_data, start_hyperopt_list, - start_hyperopt_show, start_list_exchanges, - start_list_hyperopts, start_list_markets, - start_list_strategies, start_list_timeframes, - start_new_hyperopt, start_new_strategy, - start_show_trades, start_test_pairlist, +from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data, + start_hyperopt_list, start_hyperopt_show, start_list_data, + start_list_exchanges, start_list_hyperopts, start_list_markets, + start_list_strategies, start_list_timeframes, start_new_hyperopt, + start_new_strategy, start_show_trades, start_test_pairlist, start_trading) from freqtrade.configuration import setup_utils_configuration from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, - patch_exchange, +from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) +from tests.conftest_trades import MOCK_TRADE_COUNT def test_setup_utils_configuration(): @@ -436,6 +435,16 @@ def test_list_markets(mocker, markets, capsys): assert re.search(r"^BLK/BTC$", captured.out, re.MULTILINE) assert re.search(r"^LTC/USD$", captured.out, re.MULTILINE) + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(side_effect=ValueError)) + # Test --one-column + args = [ + "list-markets", + '--config', 'config.json.example', + "--one-column" + ] + with pytest.raises(OperationalException, match=r"Cannot get markets.*"): + start_list_markets(get_args(args), False) + def test_create_datadir_failed(caplog): @@ -477,6 +486,12 @@ def test_start_new_strategy(mocker, caplog): assert "CoolNewStrategy" in wt_mock.call_args_list[0][0][0] assert log_has_re("Writing strategy to .*", caplog) + mocker.patch('freqtrade.commands.deploy_commands.setup_utils_configuration') + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + with pytest.raises(OperationalException, + match=r".* already exists. Please choose another Strategy Name\."): + start_new_strategy(get_args(args)) + def test_start_new_strategy_DefaultStrat(mocker, caplog): args = [ @@ -513,6 +528,12 @@ def test_start_new_hyperopt(mocker, caplog): assert "CoolNewhyperopt" in wt_mock.call_args_list[0][0][0] assert log_has_re("Writing hyperopt to .*", caplog) + mocker.patch('freqtrade.commands.deploy_commands.setup_utils_configuration') + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + with pytest.raises(OperationalException, + match=r".* already exists. Please choose another Hyperopt Name\."): + start_new_hyperopt(get_args(args)) + def test_start_new_hyperopt_DefaultHyperopt(mocker, caplog): args = [ @@ -552,6 +573,51 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): assert dl_mock.call_count == 1 +def test_download_data_timerange(mocker, caplog, markets): + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', + MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) + ) + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + "--timerange", "20200101-" + ] + with pytest.raises(OperationalException, + match=r"--days and --timerange are mutually.*"): + start_download_data(get_args(args)) + assert dl_mock.call_count == 0 + + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + ] + start_download_data(get_args(args)) + assert dl_mock.call_count == 1 + # 20days ago + days_ago = arrow.get(arrow.utcnow().shift(days=-20).date()).int_timestamp + assert dl_mock.call_args_list[0][1]['timerange'].startts == days_ago + + dl_mock.reset_mock() + args = [ + "download-data", + "--exchange", "binance", + "--pairs", "ETH/BTC", "XRP/BTC", + "--timerange", "20200101-" + ] + start_download_data(get_args(args)) + assert dl_mock.call_count == 1 + + assert dl_mock.call_args_list[0][1]['timerange'].startts == arrow.Arrow( + 2020, 1, 1).int_timestamp + + def test_download_data_no_markets(mocker, caplog): dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) @@ -652,6 +718,7 @@ def test_start_list_strategies(mocker, caplog, capsys): "list-strategies", "--strategy-path", str(Path(__file__).parent.parent / "strategy" / "strats"), + '--no-color', ] pargs = get_args(args) # pargs['config'] = None @@ -667,7 +734,7 @@ def test_start_list_hyperopts(mocker, caplog, capsys): args = [ "list-hyperopts", "--hyperopt-path", - str(Path(__file__).parent.parent / "optimize"), + str(Path(__file__).parent.parent / "optimize" / "hyperopts"), "-1" ] pargs = get_args(args) @@ -683,7 +750,7 @@ def test_start_list_hyperopts(mocker, caplog, capsys): args = [ "list-hyperopts", "--hyperopt-path", - str(Path(__file__).parent.parent / "optimize"), + str(Path(__file__).parent.parent / "optimize" / "hyperopts"), ] pargs = get_args(args) # pargs['config'] = None @@ -692,7 +759,6 @@ def test_start_list_hyperopts(mocker, caplog, capsys): assert "TestHyperoptLegacy" not in captured.out assert "legacy_hyperopt.py" not in captured.out assert "DefaultHyperOpt" in captured.out - assert "test_hyperopt.py" in captured.out def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): @@ -727,6 +793,25 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): assert re.match(r"Pairs for .*", captured.out) assert re.match("['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC', 'XRP/BTC']", captured.out) + args = [ + 'test-pairlist', + '-c', 'config.json.example', + '--one-column', + ] + start_test_pairlist(get_args(args)) + captured = capsys.readouterr() + assert re.match(r"ETH/BTC\nTKN/BTC\nBLK/BTC\nLTC/BTC\nXRP/BTC\n", captured.out) + + args = [ + 'test-pairlist', + '-c', 'config.json.example', + '--print-json', + ] + start_test_pairlist(get_args(args)) + captured = capsys.readouterr() + assert re.match(r'Pairs for BTC: \n\["ETH/BTC","TKN/BTC","BLK/BTC","LTC/BTC","XRP/BTC"\]\n', + captured.out) + def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): mocker.patch( @@ -736,7 +821,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): args = [ "hyperopt-list", - "--no-details" + "--no-details", ] pargs = get_args(args) pargs['config'] = None @@ -749,7 +834,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): args = [ "hyperopt-list", "--best", - "--no-details" + "--no-details", ] pargs = get_args(args) pargs['config'] = None @@ -763,7 +848,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): args = [ "hyperopt-list", "--profitable", - "--no-details" + "--no-details", ] pargs = get_args(args) pargs['config'] = None @@ -776,7 +861,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): " 11/12", " 12/12"]) args = [ "hyperopt-list", - "--profitable" + "--profitable", ] pargs = get_args(args) pargs['config'] = None @@ -792,7 +877,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): "hyperopt-list", "--no-details", "--no-color", - "--min-trades", "20" + "--min-trades", "20", ] pargs = get_args(args) pargs['config'] = None @@ -806,7 +891,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): "hyperopt-list", "--profitable", "--no-details", - "--max-trades", "20" + "--max-trades", "20", ] pargs = get_args(args) pargs['config'] = None @@ -821,7 +906,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): "hyperopt-list", "--profitable", "--no-details", - "--min-avg-profit", "0.11" + "--min-avg-profit", "0.11", ] pargs = get_args(args) pargs['config'] = None @@ -835,7 +920,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): args = [ "hyperopt-list", "--no-details", - "--max-avg-profit", "0.10" + "--max-avg-profit", "0.10", ] pargs = get_args(args) pargs['config'] = None @@ -849,7 +934,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): args = [ "hyperopt-list", "--no-details", - "--min-total-profit", "0.4" + "--min-total-profit", "0.4", ] pargs = get_args(args) pargs['config'] = None @@ -863,7 +948,35 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): args = [ "hyperopt-list", "--no-details", - "--max-total-profit", "0.4" + "--max-total-profit", "0.4", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 4/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--min-objective", "0.1", + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--max-objective", "0.1", ] pargs = get_args(args) pargs['config'] = None @@ -878,7 +991,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): "hyperopt-list", "--profitable", "--no-details", - "--min-avg-time", "2000" + "--min-avg-time", "2000", ] pargs = get_args(args) pargs['config'] = None @@ -892,7 +1005,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): args = [ "hyperopt-list", "--no-details", - "--max-avg-time", "1500" + "--max-avg-time", "1500", ] pargs = get_args(args) pargs['config'] = None @@ -906,7 +1019,7 @@ def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): args = [ "hyperopt-list", "--no-details", - "--export-csv", "test_file.csv" + "--export-csv", "test_file.csv", ] pargs = get_args(args) pargs['config'] = None @@ -1043,9 +1156,43 @@ def test_convert_data_trades(mocker, testdatadir): assert trades_mock.call_args[1]['erase'] is False +def test_start_list_data(testdatadir, capsys): + args = [ + "list-data", + "--data-format-ohlcv", + "json", + "--datadir", + str(testdatadir), + ] + pargs = get_args(args) + pargs['config'] = None + start_list_data(pargs) + captured = capsys.readouterr() + assert "Found 16 pair / timeframe combinations." in captured.out + assert "\n| Pair | Timeframe |\n" in captured.out + assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m |\n" in captured.out + + args = [ + "list-data", + "--data-format-ohlcv", + "json", + "--pairs", "XRP/ETH", + "--datadir", + str(testdatadir), + ] + pargs = get_args(args) + pargs['config'] = None + start_list_data(pargs) + captured = capsys.readouterr() + assert "Found 2 pair / timeframe combinations." in captured.out + assert "\n| Pair | Timeframe |\n" in captured.out + assert "UNITTEST/BTC" not in captured.out + assert "\n| XRP/ETH | 1m, 5m |\n" in captured.out + + @pytest.mark.usefixtures("init_persistence") def test_show_trades(mocker, fee, capsys, caplog): - mocker.patch("freqtrade.persistence.init") + mocker.patch("freqtrade.persistence.init_db") create_mock_trades(fee) args = [ "show-trades", @@ -1055,7 +1202,7 @@ def test_show_trades(mocker, fee, capsys, caplog): pargs = get_args(args) pargs['config'] = None start_show_trades(pargs) - assert log_has("Printing 3 Trades: ", caplog) + assert log_has(f"Printing {MOCK_TRADE_COUNT} Trades: ", caplog) captured = capsys.readouterr() assert "Trade(id=1" in captured.out assert "Trade(id=2" in captured.out diff --git a/tests/config_test_comments.json b/tests/config_test_comments.json index d9d4a86bc..4f201f86c 100644 --- a/tests/config_test_comments.json +++ b/tests/config_test_comments.json @@ -9,7 +9,7 @@ "fiat_display_currency": "USD", // C++-style comment "amount_reserve_percent" : 0.05, // And more, tabs before this comment "dry_run": false, - "ticker_interval": "5m", + "timeframe": "5m", "trailing_stop": false, "trailing_stop_positive": 0.005, "trailing_stop_positive_offset": 0.0051, diff --git a/tests/conftest.py b/tests/conftest.py index 1cd3bcc0f..965980f7a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,15 +13,18 @@ import numpy as np import pytest from telegram import Chat, Message, Update -from freqtrade import constants, persistence +from freqtrade import constants from freqtrade.commands import Arguments from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.persistence import Trade +from freqtrade.persistence import Trade, init_db from freqtrade.resolvers import ExchangeResolver from freqtrade.worker import Worker +from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4, + mock_trade_5, mock_trade_6) + logging.getLogger('').setLevel(logging.INFO) @@ -56,6 +59,7 @@ def patched_configuration_load_config_file(mocker, config) -> None: def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> None: + mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) @@ -77,7 +81,7 @@ def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> No def get_patched_exchange(mocker, config, api_mock=None, id='bittrex', mock_markets=True) -> Exchange: patch_exchange(mocker, api_mock, id, mock_markets) - config["exchange"]["name"] = id + config['exchange']['name'] = id try: exchange = ExchangeResolver.load_exchange(id, config) except ImportError: @@ -127,7 +131,7 @@ def patch_freqtradebot(mocker, config) -> None: :return: None """ mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) - persistence.init(config['db_url']) + init_db(config['db_url']) patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.RPCManager._init', MagicMock()) mocker.patch('freqtrade.freqtradebot.RPCManager.send_msg', MagicMock()) @@ -142,6 +146,7 @@ def get_patched_freqtradebot(mocker, config) -> FreqtradeBot: :return: FreqtradeBot """ patch_freqtradebot(mocker, config) + config['datadir'] = Path(config['datadir']) return FreqtradeBot(config) @@ -162,7 +167,7 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: :param value: which value IStrategy.get_signal() must return :return: None """ - freqtrade.strategy.get_signal = lambda e, s, t: value + freqtrade.strategy.get_signal = lambda e, s, x: value freqtrade.exchange.refresh_latest_ohlcv = lambda p: None @@ -171,44 +176,22 @@ def create_mock_trades(fee): Create some fake trades ... """ # Simulate dry_run entries - trade = Trade( - pair='ETH/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='dry_run_buy_12345' - ) + trade = mock_trade_1(fee) Trade.session.add(trade) - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - close_rate=0.128, - close_profit=0.005, - exchange='bittrex', - is_open=False, - open_order_id='dry_run_sell_12345' - ) + trade = mock_trade_2(fee) Trade.session.add(trade) - # Simulate prod entry - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='prod_buy_12345' - ) + trade = mock_trade_3(fee) + Trade.session.add(trade) + + trade = mock_trade_4(fee) + Trade.session.add(trade) + + trade = mock_trade_5(fee) + Trade.session.add(trade) + + trade = mock_trade_6(fee) Trade.session.add(trade) @@ -236,7 +219,7 @@ def patch_coingekko(mocker) -> None: @pytest.fixture(scope='function') def init_persistence(default_conf): - persistence.init(default_conf['db_url'], default_conf['dry_run']) + init_db(default_conf['db_url'], default_conf['dry_run']) @pytest.fixture(scope="function") @@ -247,7 +230,7 @@ def default_conf(testdatadir): "stake_currency": "BTC", "stake_amount": 0.001, "fiat_display_currency": "USD", - "ticker_interval": '5m', + "timeframe": '5m', "dry_run": True, "cancel_open_orders_on_exit": False, "minimal_roi": { @@ -314,7 +297,7 @@ def default_conf(testdatadir): @pytest.fixture def update(): _update = Update(0) - _update.message = Message(0, 0, datetime.utcnow(), Chat(0, 0)) + _update.message = Message(0, datetime.utcnow(), Chat(0, 0)) return _update @@ -660,7 +643,8 @@ def shitcoinmarkets(markets): Fixture with shitcoin markets - used to test filters in pairlists """ shitmarkets = deepcopy(markets) - shitmarkets.update({'HOT/BTC': { + shitmarkets.update({ + 'HOT/BTC': { 'id': 'HOTBTC', 'symbol': 'HOT/BTC', 'base': 'HOT', @@ -765,7 +749,32 @@ def shitcoinmarkets(markets): "spot": True, "future": False, "active": True - }, + }, + 'ADADOUBLE/USDT': { + "percentage": True, + "tierBased": False, + "taker": 0.001, + "maker": 0.001, + "precision": { + "base": 8, + "quote": 8, + "amount": 2, + "price": 4 + }, + "limits": { + }, + "id": "ADADOUBLEUSDT", + "symbol": "ADADOUBLE/USDT", + "base": "ADADOUBLE", + "quote": "USDT", + "baseId": "ADADOUBLE", + "quoteId": "USDT", + "info": {}, + "type": "spot", + "spot": True, + "future": False, + "active": True + }, }) return shitmarkets @@ -776,21 +785,32 @@ def markets_empty(): @pytest.fixture(scope='function') -def limit_buy_order(): +def limit_buy_order_open(): return { 'id': 'mocked_limit_buy', 'type': 'limit', 'side': 'buy', 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), + 'timestamp': arrow.utcnow().int_timestamp, 'price': 0.00001099, 'amount': 90.99181073, - 'filled': 90.99181073, - 'remaining': 0.0, - 'status': 'closed' + 'filled': 0.0, + 'cost': 0.0009999, + 'remaining': 90.99181073, + 'status': 'open' } +@pytest.fixture(scope='function') +def limit_buy_order(limit_buy_order_open): + order = deepcopy(limit_buy_order_open) + order['status'] = 'closed' + order['filled'] = order['amount'] + order['remaining'] = 0.0 + return order + + @pytest.fixture(scope='function') def market_buy_order(): return { @@ -891,7 +911,7 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': None, - 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp, 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', @@ -912,7 +932,7 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': 'AZNPFF-4AC4N-7MKTAT', 'clientOrderId': None, - 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp, 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'lastTradeTimestamp': None, 'status': 'canceled', @@ -933,7 +953,7 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': 'alb1234123', - 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp, 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', @@ -954,7 +974,7 @@ def limit_buy_order_canceled_empty(request): 'info': {}, 'id': '1234512345', 'clientOrderId': 'alb1234123', - 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'timestamp': arrow.utcnow().shift(minutes=-601).int_timestamp, 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'lastTradeTimestamp': None, 'symbol': 'LTC/USDT', @@ -973,21 +993,31 @@ def limit_buy_order_canceled_empty(request): @pytest.fixture -def limit_sell_order(): +def limit_sell_order_open(): return { 'id': 'mocked_limit_sell', 'type': 'limit', 'side': 'sell', 'pair': 'mocked', 'datetime': arrow.utcnow().isoformat(), + 'timestamp': arrow.utcnow().int_timestamp, 'price': 0.00001173, 'amount': 90.99181073, - 'filled': 90.99181073, - 'remaining': 0.0, - 'status': 'closed' + 'filled': 0.0, + 'remaining': 90.99181073, + 'status': 'open' } +@pytest.fixture +def limit_sell_order(limit_sell_order_open): + order = deepcopy(limit_sell_order_open) + order['remaining'] = 0.0 + order['filled'] = order['amount'] + order['status'] = 'closed' + return order + + @pytest.fixture def order_book_l2(): return MagicMock(return_value={ @@ -1054,7 +1084,7 @@ def ohlcv_history_list(): @pytest.fixture def ohlcv_history(ohlcv_history_list): return ohlcv_to_dataframe(ohlcv_history_list, "5m", pair="UNITTEST/BTC", - fill_missing=True) + fill_missing=True, drop_incomplete=False) @pytest.fixture @@ -1386,6 +1416,28 @@ def tickers(): "quoteVolume": 0.0, "info": {} }, + "ADADOUBLE/USDT": { + "symbol": "ADADOUBLE/USDT", + "timestamp": 1580469388244, + "datetime": "2020-01-31T11:16:28.244Z", + "high": None, + "low": None, + "bid": 0.7305, + "bidVolume": None, + "ask": 0.7342, + "askVolume": None, + "vwap": None, + "open": None, + "close": None, + "last": 0, + "previousClose": None, + "change": None, + "percentage": 2.628, + "average": None, + "baseVolume": 0.0, + "quoteVolume": 0.0, + "info": {} + }, }) @@ -1423,7 +1475,7 @@ def trades_for_order(): @pytest.fixture(scope="function") def trades_history(): - return [[1565798399463, '126181329', None, 'buy', 0.019627, 0.04, 0.00078508], + return [[1565798389463, '126181329', None, 'buy', 0.019627, 0.04, 0.00078508], [1565798399629, '126181330', None, 'buy', 0.019627, 0.244, 0.004788987999999999], [1565798399752, '126181331', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], [1565798399862, '126181332', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], @@ -1536,16 +1588,7 @@ def fetch_trades_result(): @pytest.fixture(scope="function") def trades_for_order2(): - return [{'info': {'id': 34567, - 'orderId': 123456, - 'price': '0.24544100', - 'qty': '8.00000000', - 'commission': '0.00800000', - 'commissionAsset': 'LTC', - 'time': 1521663363189, - 'isBuyer': True, - 'isMaker': False, - 'isBestMatch': True}, + return [{'info': {}, 'timestamp': 1521663363189, 'datetime': '2018-03-21T20:16:03.189Z', 'symbol': 'LTC/ETH', @@ -1557,16 +1600,7 @@ def trades_for_order2(): 'cost': 1.963528, 'amount': 4.0, 'fee': {'cost': 0.004, 'currency': 'LTC'}}, - {'info': {'id': 34567, - 'orderId': 123456, - 'price': '0.24544100', - 'qty': '8.00000000', - 'commission': '0.00800000', - 'commissionAsset': 'LTC', - 'time': 1521663363189, - 'isBuyer': True, - 'isMaker': False, - 'isBestMatch': True}, + {'info': {}, 'timestamp': 1521663363189, 'datetime': '2018-03-21T20:16:03.189Z', 'symbol': 'LTC/ETH', @@ -1580,6 +1614,14 @@ def trades_for_order2(): 'fee': {'cost': 0.004, 'currency': 'LTC'}}] +@pytest.fixture(scope="function") +def trades_for_order3(trades_for_order2): + # Different fee currencies for each trade + trades_for_order = deepcopy(trades_for_order2) + trades_for_order[0]['fee'] = {'cost': 0.02, 'currency': 'BNB'} + return trades_for_order + + @pytest.fixture def buy_order_fee(): return { diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py new file mode 100644 index 000000000..e84722041 --- /dev/null +++ b/tests/conftest_trades.py @@ -0,0 +1,287 @@ +from datetime import datetime, timedelta, timezone + +from freqtrade.persistence.models import Order, Trade + + +MOCK_TRADE_COUNT = 6 + + +def mock_order_1(): + return { + 'id': '1234', + 'symbol': 'ETH/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_trade_1(fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='dry_run_buy_12345', + strategy='DefaultStrategy', + ) + o = Order.parse_from_ccxt_object(mock_order_1(), 'ETH/BTC', 'buy') + trade.orders.append(o) + return trade + + +def mock_order_2(): + return { + 'id': '1235', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_order_2_sell(): + return { + 'id': '12366', + 'symbol': 'ETC/BTC', + 'status': 'closed', + 'side': 'sell', + 'type': 'limit', + 'price': 0.128, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_trade_2(fee): + """ + Closed trade... + """ + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + close_rate=0.128, + close_profit=0.005, + exchange='bittrex', + is_open=False, + open_order_id='dry_run_sell_12345', + strategy='DefaultStrategy', + sell_reason='sell_signal', + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc), + ) + o = Order.parse_from_ccxt_object(mock_order_2(), 'ETC/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_2_sell(), 'ETC/BTC', 'sell') + trade.orders.append(o) + return trade + + +def mock_order_3(): + return { + 'id': '41231a12a', + 'symbol': 'XRP/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.05, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_order_3_sell(): + return { + 'id': '41231a666a', + 'symbol': 'XRP/BTC', + 'status': 'closed', + 'side': 'sell', + 'type': 'stop_loss_limit', + 'price': 0.06, + 'average': 0.06, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_trade_3(fee): + """ + Closed trade + """ + trade = Trade( + pair='XRP/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.05, + close_rate=0.06, + close_profit=0.01, + exchange='bittrex', + is_open=False, + sell_reason='roi', + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc), + ) + o = Order.parse_from_ccxt_object(mock_order_3(), 'XRP/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_3_sell(), 'XRP/BTC', 'sell') + trade.orders.append(o) + return trade + + +def mock_order_4(): + return { + 'id': 'prod_buy_12345', + 'symbol': 'ETC/BTC', + 'status': 'open', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 0.0, + 'remaining': 123.0, + } + + +def mock_trade_4(fee): + """ + Simulate prod entry + """ + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=124.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='prod_buy_12345', + strategy='DefaultStrategy', + ) + o = Order.parse_from_ccxt_object(mock_order_4(), 'ETC/BTC', 'buy') + trade.orders.append(o) + return trade + + +def mock_order_5(): + return { + 'id': 'prod_buy_3455', + 'symbol': 'XRP/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 123.0, + 'remaining': 0.0, + } + + +def mock_order_5_stoploss(): + return { + 'id': 'prod_stoploss_3455', + 'symbol': 'XRP/BTC', + 'status': 'open', + 'side': 'sell', + 'type': 'stop_loss_limit', + 'price': 0.123, + 'amount': 123.0, + 'filled': 0.0, + 'remaining': 123.0, + } + + +def mock_trade_5(fee): + """ + Simulate prod entry with stoploss + """ + trade = Trade( + pair='XRP/BTC', + stake_amount=0.001, + amount=123.0, + amount_requested=124.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + strategy='SampleStrategy', + stoploss_order_id='prod_stoploss_3455' + ) + o = Order.parse_from_ccxt_object(mock_order_5(), 'XRP/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_5_stoploss(), 'XRP/BTC', 'stoploss') + trade.orders.append(o) + return trade + + +def mock_order_6(): + return { + 'id': 'prod_buy_6', + 'symbol': 'LTC/BTC', + 'status': 'closed', + 'side': 'buy', + 'type': 'limit', + 'price': 0.15, + 'amount': 2.0, + 'filled': 2.0, + 'remaining': 0.0, + } + + +def mock_order_6_sell(): + return { + 'id': 'prod_sell_6', + 'symbol': 'LTC/BTC', + 'status': 'open', + 'side': 'sell', + 'type': 'limit', + 'price': 0.20, + 'amount': 2.0, + 'filled': 0.0, + 'remaining': 2.0, + } + + +def mock_trade_6(fee): + """ + Simulate prod entry with open sell order + """ + trade = Trade( + pair='LTC/BTC', + stake_amount=0.001, + amount=2.0, + amount_requested=2.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.15, + exchange='bittrex', + strategy='SampleStrategy', + open_order_id="prod_sell_6", + ) + o = Order.parse_from_ccxt_object(mock_order_6(), 'LTC/BTC', 'buy') + trade.orders.append(o) + o = Order.parse_from_ccxt_object(mock_order_6_sell(), 'LTC/BTC', 'sell') + trade.orders.append(o) + return trade diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index b65db7fd8..1592fac10 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -6,24 +6,56 @@ from arrow import Arrow from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange -from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, - analyze_trade_parallelism, - calculate_max_drawdown, - combine_dataframes_with_mean, - create_cum_profit, - extract_trades_of_period, - load_backtest_data, load_trades, +from freqtrade.constants import LAST_BT_RESULT_FN +from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, analyze_trade_parallelism, + calculate_market_change, calculate_max_drawdown, + combine_dataframes_with_mean, create_cum_profit, + extract_trades_of_period, get_latest_backtest_filename, + get_latest_hyperopt_file, load_backtest_data, load_trades, load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history +from freqtrade.optimize.backtesting import BacktestResult from tests.conftest import create_mock_trades +from tests.conftest_trades import MOCK_TRADE_COUNT -def test_load_backtest_data(testdatadir): +def test_get_latest_backtest_filename(testdatadir, mocker): + with pytest.raises(ValueError, match=r"Directory .* does not exist\."): + get_latest_backtest_filename(testdatadir / 'does_not_exist') + + with pytest.raises(ValueError, + match=r"Directory .* does not seem to contain .*"): + get_latest_backtest_filename(testdatadir.parent) + + res = get_latest_backtest_filename(testdatadir) + assert res == 'backtest-result_new.json' + + res = get_latest_backtest_filename(str(testdatadir)) + assert res == 'backtest-result_new.json' + + mocker.patch("freqtrade.data.btanalysis.json_load", return_value={}) + + with pytest.raises(ValueError, match=r"Invalid '.last_result.json' format."): + get_latest_backtest_filename(testdatadir) + + +def test_get_latest_hyperopt_file(testdatadir, mocker): + res = get_latest_hyperopt_file(testdatadir / 'does_not_exist', 'testfile.pickle') + assert res == testdatadir / 'does_not_exist/testfile.pickle' + + res = get_latest_hyperopt_file(testdatadir.parent) + assert res == testdatadir.parent / "hyperopt_results.pickle" + + res = get_latest_hyperopt_file(str(testdatadir.parent)) + assert res == testdatadir.parent / "hyperopt_results.pickle" + + +def test_load_backtest_data_old_format(testdatadir): filename = testdatadir / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) - assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit"] + assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit_abs"] assert len(bt_data) == 179 # Test loading from string (must yield same result) @@ -34,24 +66,71 @@ def test_load_backtest_data(testdatadir): load_backtest_data(str("filename") + "nofile") +def test_load_backtest_data_new_format(testdatadir): + + filename = testdatadir / "backtest-result_new.json" + bt_data = load_backtest_data(filename) + assert isinstance(bt_data, DataFrame) + assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"]) + assert len(bt_data) == 179 + + # Test loading from string (must yield same result) + bt_data2 = load_backtest_data(str(filename)) + assert bt_data.equals(bt_data2) + + # Test loading from folder (must yield same result) + bt_data3 = load_backtest_data(testdatadir) + assert bt_data.equals(bt_data3) + + with pytest.raises(ValueError, match=r"File .* does not exist\."): + load_backtest_data(str("filename") + "nofile") + + with pytest.raises(ValueError, match=r"Unknown dataformat."): + load_backtest_data(testdatadir / LAST_BT_RESULT_FN) + + +def test_load_backtest_data_multi(testdatadir): + + filename = testdatadir / "backtest-result_multistrat.json" + for strategy in ('DefaultStrategy', 'TestStrategy'): + bt_data = load_backtest_data(filename, strategy=strategy) + assert isinstance(bt_data, DataFrame) + assert set(bt_data.columns) == set(list(BacktestResult._fields) + ["profit_abs"]) + assert len(bt_data) == 179 + + # Test loading from string (must yield same result) + bt_data2 = load_backtest_data(str(filename), strategy=strategy) + assert bt_data.equals(bt_data2) + + with pytest.raises(ValueError, match=r"Strategy XYZ not available in the backtest result\."): + load_backtest_data(filename, strategy='XYZ') + + with pytest.raises(ValueError, match=r"Detected backtest result with more than one strategy.*"): + load_backtest_data(filename) + + @pytest.mark.usefixtures("init_persistence") def test_load_trades_from_db(default_conf, fee, mocker): create_mock_trades(fee) # remove init so it does not init again - init_mock = mocker.patch('freqtrade.persistence.init', MagicMock()) + init_mock = mocker.patch('freqtrade.data.btanalysis.init_db', MagicMock()) trades = load_trades_from_db(db_url=default_conf['db_url']) assert init_mock.call_count == 1 - assert len(trades) == 3 + assert len(trades) == MOCK_TRADE_COUNT assert isinstance(trades, DataFrame) assert "pair" in trades.columns - assert "open_time" in trades.columns + assert "open_date" in trades.columns assert "profit_percent" in trades.columns for col in BT_DATA_COLUMNS: if col not in ['index', 'open_at_end']: assert col in trades.columns + trades = load_trades_from_db(db_url=default_conf['db_url'], strategy='DefaultStrategy') + assert len(trades) == 3 + trades = load_trades_from_db(db_url=default_conf['db_url'], strategy='NoneStrategy') + assert len(trades) == 0 def test_extract_trades_of_period(testdatadir): @@ -66,13 +145,13 @@ def test_extract_trades_of_period(testdatadir): {'pair': [pair, pair, pair, pair], 'profit_percent': [0.0, 0.1, -0.2, -0.5], 'profit_abs': [0.0, 1, -2, -5], - 'open_time': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime, + 'open_date': to_datetime([Arrow(2017, 11, 13, 15, 40, 0).datetime, Arrow(2017, 11, 14, 9, 41, 0).datetime, Arrow(2017, 11, 14, 14, 20, 0).datetime, Arrow(2017, 11, 15, 3, 40, 0).datetime, ], utc=True ), - 'close_time': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime, + 'close_date': to_datetime([Arrow(2017, 11, 13, 16, 40, 0).datetime, Arrow(2017, 11, 14, 10, 41, 0).datetime, Arrow(2017, 11, 14, 15, 25, 0).datetime, Arrow(2017, 11, 15, 3, 55, 0).datetime, @@ -81,10 +160,10 @@ def test_extract_trades_of_period(testdatadir): trades1 = extract_trades_of_period(data, trades) # First and last trade are dropped as they are out of range assert len(trades1) == 2 - assert trades1.iloc[0].open_time == Arrow(2017, 11, 14, 9, 41, 0).datetime - assert trades1.iloc[0].close_time == Arrow(2017, 11, 14, 10, 41, 0).datetime - assert trades1.iloc[-1].open_time == Arrow(2017, 11, 14, 14, 20, 0).datetime - assert trades1.iloc[-1].close_time == Arrow(2017, 11, 14, 15, 25, 0).datetime + assert trades1.iloc[0].open_date == Arrow(2017, 11, 14, 9, 41, 0).datetime + assert trades1.iloc[0].close_date == Arrow(2017, 11, 14, 10, 41, 0).datetime + assert trades1.iloc[-1].open_date == Arrow(2017, 11, 14, 14, 20, 0).datetime + assert trades1.iloc[-1].close_date == Arrow(2017, 11, 14, 15, 25, 0).datetime def test_analyze_trade_parallelism(default_conf, mocker, testdatadir): @@ -105,7 +184,8 @@ def test_load_trades(default_conf, mocker): load_trades("DB", db_url=default_conf.get('db_url'), exportfilename=default_conf.get('exportfilename'), - no_trades=False + no_trades=False, + strategy="DefaultStrategy", ) assert db_mock.call_count == 1 @@ -135,6 +215,14 @@ def test_load_trades(default_conf, mocker): assert bt_mock.call_count == 0 +def test_calculate_market_change(testdatadir): + pairs = ["ETH/BTC", "ADA/BTC"] + data = load_data(datadir=testdatadir, pairs=pairs, timeframe='5m') + result = calculate_market_change(data) + assert isinstance(result, float) + assert pytest.approx(result) == 0.00955514 + + def test_combine_dataframes_with_mean(testdatadir): pairs = ["ETH/BTC", "ADA/BTC"] data = load_data(datadir=testdatadir, pairs=pairs, timeframe='5m') @@ -165,7 +253,7 @@ def test_create_cum_profit1(testdatadir): filename = testdatadir / "backtest-result_test.json" bt_data = load_backtest_data(filename) # Move close-time to "off" the candle, to make sure the logic still works - bt_data.loc[:, 'close_time'] = bt_data.loc[:, 'close_time'] + DateOffset(seconds=20) + bt_data.loc[:, 'close_date'] = bt_data.loc[:, 'close_date'] + DateOffset(seconds=20) timerange = TimeRange.parse_timerange("20180110-20180112") df = load_pair_history(pair="TRX/BTC", timeframe='5m', @@ -204,11 +292,11 @@ def test_calculate_max_drawdown2(): -0.033961, 0.010680, 0.010886, -0.029274, 0.011178, 0.010693, 0.010711] dates = [Arrow(2020, 1, 1).shift(days=i) for i in range(len(values))] - df = DataFrame(zip(values, dates), columns=['profit', 'open_time']) + df = DataFrame(zip(values, dates), columns=['profit', 'open_date']) # sort by profit and reset index df = df.sort_values('profit').reset_index(drop=True) df1 = df.copy() - drawdown, h, low = calculate_max_drawdown(df, date_col='open_time', value_col='profit') + drawdown, h, low = calculate_max_drawdown(df, date_col='open_date', value_col='profit') # Ensure df has not been altered. assert df.equals(df1) @@ -217,6 +305,6 @@ def test_calculate_max_drawdown2(): assert h < low assert drawdown == 0.091755 - df = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_time']) + df = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_date']) with pytest.raises(ValueError, match='No losing trade, therefore no drawdown.'): - calculate_max_drawdown(df, date_col='open_time', value_col='profit') + calculate_max_drawdown(df, date_col='open_date', value_col='profit') diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 4a580366f..4fdcce4d2 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -1,14 +1,15 @@ # pragma pylint: disable=missing-docstring, C0103 import logging +import pytest + from freqtrade.configuration.timerange import TimeRange -from freqtrade.data.converter import (convert_ohlcv_format, - convert_trades_format, - ohlcv_fill_up_missing_data, - ohlcv_to_dataframe, trades_dict_to_list, - trades_remove_duplicates, trim_dataframe) -from freqtrade.data.history import (get_timerange, load_data, - load_pair_history, validate_backtest_data) +from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, + ohlcv_fill_up_missing_data, ohlcv_to_dataframe, + trades_dict_to_list, trades_remove_duplicates, + trades_to_ohlcv, trim_dataframe) +from freqtrade.data.history import (get_timerange, load_data, load_pair_history, + validate_backtest_data) from tests.conftest import log_has from tests.data.test_history import _backup_file, _clean_test_file @@ -28,6 +29,28 @@ def test_ohlcv_to_dataframe(ohlcv_history_list, caplog): assert log_has('Converting candle (OHLCV) data to dataframe for pair UNITTEST/BTC.', caplog) +def test_trades_to_ohlcv(ohlcv_history_list, caplog): + + caplog.set_level(logging.DEBUG) + with pytest.raises(ValueError, match="Trade-list empty."): + trades_to_ohlcv([], '1m') + + trades = [ + [1570752011620, "13519807", None, "sell", 0.00141342, 23.0, 0.03250866], + [1570752011620, "13519808", None, "sell", 0.00141266, 54.0, 0.07628364], + [1570752017964, "13519809", None, "sell", 0.00141266, 8.0, 0.01130128]] + + df = trades_to_ohlcv(trades, '1m') + assert not df.empty + assert len(df) == 1 + assert 'open' in df.columns + assert 'high' in df.columns + assert 'low' in df.columns + assert 'close' in df.columns + assert df.loc[:, 'high'][0] == 0.00141342 + assert df.loc[:, 'low'][0] == 0.00141266 + + def test_ohlcv_fill_up_missing_data(testdatadir, caplog): data = load_pair_history(datadir=testdatadir, timeframe='1m', diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index c2d6e82f1..ee2e551b6 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,18 +1,19 @@ +from datetime import datetime, timezone from unittest.mock import MagicMock -from pandas import DataFrame import pytest +from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider -from freqtrade.pairlist.pairlistmanager import PairListManager -from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.exceptions import ExchangeError, OperationalException +from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.state import RunMode from tests.conftest import get_patched_exchange def test_ohlcv(mocker, default_conf, ohlcv_history): default_conf["runmode"] = RunMode.DRY_RUN - timeframe = default_conf["ticker_interval"] + timeframe = default_conf["timeframe"] exchange = get_patched_exchange(mocker, default_conf) exchange._klines[("XRP/BTC", timeframe)] = ohlcv_history exchange._klines[("UNITTEST/BTC", timeframe)] = ohlcv_history @@ -51,49 +52,74 @@ def test_historic_ohlcv(mocker, default_conf, ohlcv_history): assert historymock.call_args_list[0][1]["timeframe"] == "5m" +def test_historic_ohlcv_dataformat(mocker, default_conf, ohlcv_history): + hdf5loadmock = MagicMock(return_value=ohlcv_history) + jsonloadmock = MagicMock(return_value=ohlcv_history) + mocker.patch("freqtrade.data.history.hdf5datahandler.HDF5DataHandler._ohlcv_load", hdf5loadmock) + mocker.patch("freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load", jsonloadmock) + + default_conf["runmode"] = RunMode.BACKTEST + exchange = get_patched_exchange(mocker, default_conf) + dp = DataProvider(default_conf, exchange) + data = dp.historic_ohlcv("UNITTEST/BTC", "5m") + assert isinstance(data, DataFrame) + hdf5loadmock.assert_not_called() + jsonloadmock.assert_called_once() + + # Swiching to dataformat hdf5 + hdf5loadmock.reset_mock() + jsonloadmock.reset_mock() + default_conf["dataformat_ohlcv"] = "hdf5" + dp = DataProvider(default_conf, exchange) + data = dp.historic_ohlcv("UNITTEST/BTC", "5m") + assert isinstance(data, DataFrame) + hdf5loadmock.assert_called_once() + jsonloadmock.assert_not_called() + + def test_get_pair_dataframe(mocker, default_conf, ohlcv_history): default_conf["runmode"] = RunMode.DRY_RUN - ticker_interval = default_conf["ticker_interval"] + timeframe = default_conf["timeframe"] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines[("XRP/BTC", ticker_interval)] = ohlcv_history - exchange._klines[("UNITTEST/BTC", ticker_interval)] = ohlcv_history + exchange._klines[("XRP/BTC", timeframe)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", timeframe)] = ohlcv_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN - assert ohlcv_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)) - assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) - assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval) is not ohlcv_history - assert not dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval).empty - assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty + assert ohlcv_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", timeframe)) + assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", timeframe), DataFrame) + assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe) is not ohlcv_history + assert not dp.get_pair_dataframe("UNITTEST/BTC", timeframe).empty + assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe).empty # Test with and without parameter - assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)\ + assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe)\ .equals(dp.get_pair_dataframe("UNITTEST/BTC")) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.LIVE - assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) - assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty + assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", timeframe), DataFrame) + assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe).empty historymock = MagicMock(return_value=ohlcv_history) mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) default_conf["runmode"] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST - assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) - # assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty + assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", timeframe), DataFrame) + # assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe).empty def test_available_pairs(mocker, default_conf, ohlcv_history): exchange = get_patched_exchange(mocker, default_conf) - ticker_interval = default_conf["ticker_interval"] - exchange._klines[("XRP/BTC", ticker_interval)] = ohlcv_history - exchange._klines[("UNITTEST/BTC", ticker_interval)] = ohlcv_history + timeframe = default_conf["timeframe"] + exchange._klines[("XRP/BTC", timeframe)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", timeframe)] = ohlcv_history dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 - assert dp.available_pairs == [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval), ] + assert dp.available_pairs == [("XRP/BTC", timeframe), ("UNITTEST/BTC", timeframe), ] def test_refresh(mocker, default_conf, ohlcv_history): @@ -101,10 +127,10 @@ def test_refresh(mocker, default_conf, ohlcv_history): mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) exchange = get_patched_exchange(mocker, default_conf, id="binance") - ticker_interval = default_conf["ticker_interval"] - pairs = [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval)] + timeframe = default_conf["timeframe"] + pairs = [("XRP/BTC", timeframe), ("UNITTEST/BTC", timeframe)] - pairs_non_trad = [("ETH/USDT", ticker_interval), ("BTC/TUSD", "1h")] + pairs_non_trad = [("ETH/USDT", timeframe), ("BTC/TUSD", "1h")] dp = DataProvider(default_conf, exchange) dp.refresh(pairs) @@ -131,7 +157,7 @@ def test_orderbook(mocker, default_conf, order_book_l2): res = dp.orderbook('ETH/BTC', 5) assert order_book_l2.call_count == 1 assert order_book_l2.call_args_list[0][0][0] == 'ETH/BTC' - assert order_book_l2.call_args_list[0][0][1] == 5 + assert order_book_l2.call_args_list[0][0][1] >= 5 assert type(res) is dict assert 'bids' in res @@ -164,7 +190,7 @@ def test_ticker(mocker, default_conf, tickers): assert 'symbol' in res assert res['symbol'] == 'ETH/BTC' - ticker_mock = MagicMock(side_effect=DependencyException('Pair not found')) + ticker_mock = MagicMock(side_effect=ExchangeError('Pair not found')) mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock) exchange = get_patched_exchange(mocker, default_conf) dp = DataProvider(default_conf, exchange) @@ -194,3 +220,29 @@ def test_current_whitelist(mocker, default_conf, tickers): with pytest.raises(OperationalException): dp = DataProvider(default_conf, exchange) dp.current_whitelist() + + +def test_get_analyzed_dataframe(mocker, default_conf, ohlcv_history): + + default_conf["runmode"] = RunMode.DRY_RUN + + timeframe = default_conf["timeframe"] + exchange = get_patched_exchange(mocker, default_conf) + + dp = DataProvider(default_conf, exchange) + dp._set_cached_df("XRP/BTC", timeframe, ohlcv_history) + dp._set_cached_df("UNITTEST/BTC", timeframe, ohlcv_history) + + assert dp.runmode == RunMode.DRY_RUN + dataframe, time = dp.get_analyzed_dataframe("UNITTEST/BTC", timeframe) + assert ohlcv_history.equals(dataframe) + assert isinstance(time, datetime) + + dataframe, time = dp.get_analyzed_dataframe("XRP/BTC", timeframe) + assert ohlcv_history.equals(dataframe) + assert isinstance(time, datetime) + + dataframe, time = dp.get_analyzed_dataframe("NOTHING/BTC", timeframe) + assert dataframe.empty + assert isinstance(time, datetime) + assert time == datetime(1970, 1, 1, tzinfo=timezone.utc) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 6fd4d9569..99b22adda 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -12,21 +12,22 @@ from pandas import DataFrame from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange +from freqtrade.constants import AVAILABLE_DATAHANDLERS from freqtrade.data.converter import ohlcv_to_dataframe -from freqtrade.data.history.history_utils import ( - _download_pair_history, _download_trades_history, - _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange, - load_data, load_pair_history, refresh_backtest_ohlcv_data, - refresh_backtest_trades_data, refresh_data, validate_backtest_data) -from freqtrade.data.history.idatahandler import (IDataHandler, get_datahandler, - get_datahandlerclass) -from freqtrade.data.history.jsondatahandler import (JsonDataHandler, - JsonGzDataHandler) +from freqtrade.data.history.hdf5datahandler import HDF5DataHandler +from freqtrade.data.history.history_utils import (_download_pair_history, _download_trades_history, + _load_cached_data_for_updating, + convert_trades_to_ohlcv, get_timerange, load_data, + load_pair_history, refresh_backtest_ohlcv_data, + refresh_backtest_trades_data, refresh_data, + validate_backtest_data) +from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, get_datahandlerclass +from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.resolvers import StrategyResolver -from tests.conftest import (get_patched_exchange, log_has, log_has_re, - patch_exchange) +from tests.conftest import get_patched_exchange, log_has, log_has_re, patch_exchange + # Change this if modifying UNITTEST/BTC testdatafile _BTC_UNITTEST_LENGTH = 13681 @@ -36,7 +37,7 @@ def _backup_file(file: Path, copy_file: bool = False) -> None: """ Backup existing file to avoid deleting the user file :param file: complete path to the file - :param touch_file: create an empty file in replacement + :param copy_file: keep file in place too. :return: None """ file_swp = str(file) + '.swp' @@ -311,10 +312,7 @@ def test_download_backtesting_data_exception(ohlcv_history, mocker, caplog, # clean files freshly downloaded _clean_test_file(file1_1) _clean_test_file(file1_5) - assert log_has( - 'Failed to download history data for pair: "MEME/BTC", timeframe: 1m. ' - 'Error: File Error', caplog - ) + assert log_has('Failed to download history data for pair: "MEME/BTC", timeframe: 1m.', caplog) def test_load_partial_missing(testdatadir, caplog) -> None: @@ -322,7 +320,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: start = arrow.get('2018-01-01T00:00:00') end = arrow.get('2018-01-11T00:00:00') data = load_data(testdatadir, '5m', ['UNITTEST/BTC'], startup_candles=20, - timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) + timerange=TimeRange('date', 'date', start.int_timestamp, end.int_timestamp)) assert log_has( 'Using indicator startup period: 20 ...', caplog ) @@ -338,7 +336,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: start = arrow.get('2018-01-10T00:00:00') end = arrow.get('2018-02-20T00:00:00') data = load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], - timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) + timerange=TimeRange('date', 'date', start.int_timestamp, end.int_timestamp)) # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 assert td != len(data['UNITTEST/BTC']) @@ -354,7 +352,7 @@ def test_init(default_conf, mocker) -> None: assert {} == load_data( datadir=Path(''), pairs=[], - timeframe=default_conf['ticker_interval'] + timeframe=default_conf['timeframe'] ) @@ -363,13 +361,13 @@ def test_init_with_refresh(default_conf, mocker) -> None: refresh_data( datadir=Path(''), pairs=[], - timeframe=default_conf['ticker_interval'], + timeframe=default_conf['timeframe'], exchange=exchange ) assert {} == load_data( datadir=Path(''), pairs=[], - timeframe=default_conf['ticker_interval'] + timeframe=default_conf['timeframe'] ) @@ -557,6 +555,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert ght_mock.call_count == 1 # Check this in seconds - since we had to convert to seconds above too. assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time2 - 5 + assert ght_mock.call_args_list[0][1]['from_id'] is not None # clean files freshly downloaded _clean_test_file(file1) @@ -568,6 +567,27 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) + file2 = testdatadir / 'XRP_ETH-trades.json.gz' + + _backup_file(file2, True) + + ght_mock.reset_mock() + mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', + ght_mock) + # Since before first start date + since_time = int(trades_history[0][0] // 1000) - 500 + timerange = TimeRange('date', None, since_time, 0) + + assert _download_trades_history(data_handler=data_handler, exchange=exchange, + pair='XRP/ETH', timerange=timerange) + + assert ght_mock.call_count == 1 + + assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time + assert ght_mock.call_args_list[0][1]['from_id'] is None + assert log_has_re(r'Start earlier than available data. Redownloading trades for.*', caplog) + _clean_test_file(file2) + def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): @@ -597,8 +617,14 @@ def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): _clean_test_file(file1) _clean_test_file(file5) + assert not log_has('Could not convert NoDatapair to OHLCV.', caplog) -def test_jsondatahandler_ohlcv_get_pairs(testdatadir): + convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], + datadir=testdatadir, timerange=tr, erase=True) + assert log_has('Could not convert NoDatapair to OHLCV.', caplog) + + +def test_datahandler_ohlcv_get_pairs(testdatadir): pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m') # Convert to set to avoid failures due to sorting assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC', @@ -608,6 +634,25 @@ def test_jsondatahandler_ohlcv_get_pairs(testdatadir): pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m') assert set(pairs) == {'UNITTEST/BTC'} + pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m') + assert set(pairs) == {'UNITTEST/BTC'} + + +def test_datahandler_ohlcv_get_available_data(testdatadir): + paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir) + # Convert to set to avoid failures due to sorting + assert set(paircombs) == {('UNITTEST/BTC', '5m'), ('ETH/BTC', '5m'), ('XLM/BTC', '5m'), + ('TRX/BTC', '5m'), ('LTC/BTC', '5m'), ('XMR/BTC', '5m'), + ('ZEC/BTC', '5m'), ('UNITTEST/BTC', '1m'), ('ADA/BTC', '5m'), + ('ETC/BTC', '5m'), ('NXT/BTC', '5m'), ('DASH/BTC', '5m'), + ('XRP/ETH', '1m'), ('XRP/ETH', '5m'), ('UNITTEST/BTC', '30m'), + ('UNITTEST/BTC', '8m')} + + paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir) + assert set(paircombs) == {('UNITTEST/BTC', '8m')} + paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir) + assert set(paircombs) == {('UNITTEST/BTC', '5m')} + def test_jsondatahandler_trades_get_pairs(testdatadir): pairs = JsonGzDataHandler.trades_get_pairs(testdatadir) @@ -617,15 +662,17 @@ def test_jsondatahandler_trades_get_pairs(testdatadir): def test_jsondatahandler_ohlcv_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch.object(Path, "unlink", MagicMock()) + unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) dh = JsonGzDataHandler(testdatadir) assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert unlinkmock.call_count == 0 mocker.patch.object(Path, "exists", MagicMock(return_value=True)) assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert unlinkmock.call_count == 1 -def test_jsondatahandler_trades_load(mocker, testdatadir, caplog): +def test_jsondatahandler_trades_load(testdatadir, caplog): dh = JsonGzDataHandler(testdatadir) logmsg = "Old trades format detected - converting" dh.trades_load('XRP/ETH') @@ -638,26 +685,146 @@ def test_jsondatahandler_trades_load(mocker, testdatadir, caplog): def test_jsondatahandler_trades_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch.object(Path, "unlink", MagicMock()) + unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) dh = JsonGzDataHandler(testdatadir) assert not dh.trades_purge('UNITTEST/NONEXIST') + assert unlinkmock.call_count == 0 mocker.patch.object(Path, "exists", MagicMock(return_value=True)) assert dh.trades_purge('UNITTEST/NONEXIST') + assert unlinkmock.call_count == 1 -def test_jsondatahandler_ohlcv_append(testdatadir): - dh = JsonGzDataHandler(testdatadir) +@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS) +def test_datahandler_ohlcv_append(datahandler, testdatadir, ): + dh = get_datahandler(testdatadir, datahandler) with pytest.raises(NotImplementedError): dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame()) -def test_jsondatahandler_trades_append(testdatadir): - dh = JsonGzDataHandler(testdatadir) +@pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS) +def test_datahandler_trades_append(datahandler, testdatadir): + dh = get_datahandler(testdatadir, datahandler) with pytest.raises(NotImplementedError): dh.trades_append('UNITTEST/ETH', []) +def test_hdf5datahandler_trades_get_pairs(testdatadir): + pairs = HDF5DataHandler.trades_get_pairs(testdatadir) + # Convert to set to avoid failures due to sorting + assert set(pairs) == {'XRP/ETH'} + + +def test_hdf5datahandler_trades_load(testdatadir): + dh = HDF5DataHandler(testdatadir) + trades = dh.trades_load('XRP/ETH') + assert isinstance(trades, list) + + trades1 = dh.trades_load('UNITTEST/NONEXIST') + assert trades1 == [] + # data goes from 2019-10-11 - 2019-10-13 + timerange = TimeRange.parse_timerange('20191011-20191012') + + trades2 = dh._trades_load('XRP/ETH', timerange) + assert len(trades) > len(trades2) + # Check that ID is None (If it's nan, it's wrong) + assert trades2[0][2] is None + + # unfiltered load has trades before starttime + assert len([t for t in trades if t[0] < timerange.startts * 1000]) >= 0 + # filtered list does not have trades before starttime + assert len([t for t in trades2 if t[0] < timerange.startts * 1000]) == 0 + # unfiltered load has trades after endtime + assert len([t for t in trades if t[0] > timerange.stopts * 1000]) > 0 + # filtered list does not have trades after endtime + assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0 + + +def test_hdf5datahandler_trades_store(testdatadir): + dh = HDF5DataHandler(testdatadir) + trades = dh.trades_load('XRP/ETH') + + dh.trades_store('XRP/NEW', trades) + file = testdatadir / 'XRP_NEW-trades.h5' + assert file.is_file() + # Load trades back + trades_new = dh.trades_load('XRP/NEW') + + assert len(trades_new) == len(trades) + assert trades[0][0] == trades_new[0][0] + assert trades[0][1] == trades_new[0][1] + # assert trades[0][2] == trades_new[0][2] # This is nan - so comparison does not make sense + assert trades[0][3] == trades_new[0][3] + assert trades[0][4] == trades_new[0][4] + assert trades[0][5] == trades_new[0][5] + assert trades[0][6] == trades_new[0][6] + assert trades[-1][0] == trades_new[-1][0] + assert trades[-1][1] == trades_new[-1][1] + # assert trades[-1][2] == trades_new[-1][2] # This is nan - so comparison does not make sense + assert trades[-1][3] == trades_new[-1][3] + assert trades[-1][4] == trades_new[-1][4] + assert trades[-1][5] == trades_new[-1][5] + assert trades[-1][6] == trades_new[-1][6] + + _clean_test_file(file) + + +def test_hdf5datahandler_trades_purge(mocker, testdatadir): + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) + dh = HDF5DataHandler(testdatadir) + assert not dh.trades_purge('UNITTEST/NONEXIST') + assert unlinkmock.call_count == 0 + + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + assert dh.trades_purge('UNITTEST/NONEXIST') + assert unlinkmock.call_count == 1 + + +def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir): + dh = HDF5DataHandler(testdatadir) + ohlcv = dh.ohlcv_load('UNITTEST/BTC', '5m') + assert isinstance(ohlcv, DataFrame) + assert len(ohlcv) > 0 + + file = testdatadir / 'UNITTEST_NEW-5m.h5' + assert not file.is_file() + + dh.ohlcv_store('UNITTEST/NEW', '5m', ohlcv) + assert file.is_file() + + assert not ohlcv[ohlcv['date'] < '2018-01-15'].empty + + # Data gores from 2018-01-10 - 2018-01-30 + timerange = TimeRange.parse_timerange('20180115-20180119') + + # Call private function to ensure timerange is filtered in hdf5 + ohlcv = dh._ohlcv_load('UNITTEST/BTC', '5m', timerange) + ohlcv1 = dh._ohlcv_load('UNITTEST/NEW', '5m', timerange) + assert len(ohlcv) == len(ohlcv1) + assert ohlcv.equals(ohlcv1) + assert ohlcv[ohlcv['date'] < '2018-01-15'].empty + assert ohlcv[ohlcv['date'] > '2018-01-19'].empty + + _clean_test_file(file) + + # Try loading inexisting file + ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', '5m') + assert ohlcv.empty + + +def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir): + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) + dh = HDF5DataHandler(testdatadir) + assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert unlinkmock.call_count == 0 + + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert unlinkmock.call_count == 1 + + def test_gethandlerclass(): cl = get_datahandlerclass('json') assert cl == JsonDataHandler @@ -666,6 +833,9 @@ def test_gethandlerclass(): assert cl == JsonGzDataHandler assert issubclass(cl, IDataHandler) assert issubclass(cl, JsonDataHandler) + cl = get_datahandlerclass('hdf5') + assert cl == HDF5DataHandler + assert issubclass(cl, IDataHandler) with pytest.raises(ValueError, match=r"No datahandler for .*"): get_datahandlerclass('DeadBeef') @@ -677,3 +847,6 @@ def test_get_datahandler(testdatadir): assert type(dh) == JsonGzDataHandler dh1 = get_datahandler(testdatadir, 'jsongz', dh) assert id(dh1) == id(dh) + + dh = get_datahandler(testdatadir, 'hdf5') + assert type(dh) == HDF5DataHandler diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index 163ceff4b..f25dad35b 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -10,14 +10,15 @@ import numpy as np import pytest from pandas import DataFrame, to_datetime -from freqtrade.exceptions import OperationalException from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo +from freqtrade.exceptions import OperationalException from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset) + # Cases to be tested: # 1) Open trade should be removed from the end # 2) Two complete trades within dataframe (with sell hit for all) @@ -27,7 +28,7 @@ from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, #################################################################### tests_start_time = arrow.get(2018, 10, 3) -ticker_interval_in_minute = 60 +timeframe_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} # Helpers for this test file @@ -49,7 +50,7 @@ def _build_dataframe(buy_ohlc_sell_matrice): 'date': tests_start_time.shift( minutes=( ohlc[0] * - ticker_interval_in_minute)).timestamp * + timeframe_in_minute)).int_timestamp * 1000, 'buy': ohlc[1], 'open': ohlc[2], @@ -70,7 +71,7 @@ def _build_dataframe(buy_ohlc_sell_matrice): def _time_on_candle(number): return np.datetime64(tests_start_time.shift( - minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') + minutes=(number * timeframe_in_minute)).int_timestamp * 1000, 'ms') # End helper functions @@ -163,8 +164,8 @@ def test_edge_results(edge_conf, mocker, caplog, data) -> None: for c, trade in enumerate(data.trades): res = results.iloc[c] assert res.exit_type == trade.sell_reason - assert res.open_time == _get_frame_time_from_offset(trade.open_tick).replace(tzinfo=None) - assert res.close_time == _get_frame_time_from_offset(trade.close_tick).replace(tzinfo=None) + assert res.open_date == _get_frame_time_from_offset(trade.open_tick).replace(tzinfo=None) + assert res.close_date == _get_frame_time_from_offset(trade.close_tick).replace(tzinfo=None) def test_adjust(mocker, edge_conf): @@ -250,7 +251,7 @@ def test_edge_heartbeat_calculate(mocker, edge_conf): heartbeat = edge_conf['edge']['process_throttle_secs'] # should not recalculate if heartbeat not reached - edge._last_updated = arrow.utcnow().timestamp - heartbeat + 1 + edge._last_updated = arrow.utcnow().int_timestamp - heartbeat + 1 assert edge.calculate() is False @@ -262,7 +263,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', NEOBTC = [ [ - tests_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, @@ -274,7 +275,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', base = 0.002 LTCBTC = [ [ - tests_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + tests_start_time.shift(minutes=(x * timeframe_in_minute)).int_timestamp * 1000, math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, @@ -298,7 +299,7 @@ def test_edge_process_downloaded_data(mocker, edge_conf): assert edge.calculate() assert len(edge._cached_pairs) == 2 - assert edge._last_updated <= arrow.utcnow().timestamp + 2 + assert edge._last_updated <= arrow.utcnow().int_timestamp + 2 def test_edge_process_no_data(mocker, edge_conf, caplog): @@ -354,10 +355,8 @@ def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectanc 'stoploss': -0.9, 'profit_percent': '', 'profit_abs': '', - 'open_time': np.datetime64('2018-10-03T00:05:00.000000000'), - 'close_time': np.datetime64('2018-10-03T00:10:00.000000000'), - 'open_index': 1, - 'close_index': 1, + 'open_date': np.datetime64('2018-10-03T00:05:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:10:00.000000000'), 'trade_duration': '', 'open_rate': 17, 'close_rate': 17, @@ -367,10 +366,8 @@ def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectanc 'stoploss': -0.9, 'profit_percent': '', 'profit_abs': '', - 'open_time': np.datetime64('2018-10-03T00:20:00.000000000'), - 'close_time': np.datetime64('2018-10-03T00:25:00.000000000'), - 'open_index': 4, - 'close_index': 4, + 'open_date': np.datetime64('2018-10-03T00:20:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:25:00.000000000'), 'trade_duration': '', 'open_rate': 20, 'close_rate': 20, @@ -380,10 +377,8 @@ def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectanc 'stoploss': -0.9, 'profit_percent': '', 'profit_abs': '', - 'open_time': np.datetime64('2018-10-03T00:30:00.000000000'), - 'close_time': np.datetime64('2018-10-03T00:40:00.000000000'), - 'open_index': 6, - 'close_index': 7, + 'open_date': np.datetime64('2018-10-03T00:30:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:40:00.000000000'), 'trade_duration': '', 'open_rate': 26, 'close_rate': 34, @@ -409,3 +404,156 @@ def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectanc final = edge._process_expectancy(trades_df) assert len(final) == 0 assert isinstance(final, dict) + + +def test_process_expectancy_remove_pumps(mocker, edge_conf, fee,): + edge_conf['edge']['min_trade_number'] = 2 + edge_conf['edge']['remove_pumps'] = True + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + + freqtrade.exchange.get_fee = fee + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + + trades = [ + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_date': np.datetime64('2018-10-03T00:05:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:10:00.000000000'), + 'open_index': 1, + 'close_index': 1, + 'trade_duration': '', + 'open_rate': 17, + 'close_rate': 15, + 'exit_type': 'sell_signal'}, + + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_date': np.datetime64('2018-10-03T00:20:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:25:00.000000000'), + 'open_index': 4, + 'close_index': 4, + 'trade_duration': '', + 'open_rate': 20, + 'close_rate': 10, + 'exit_type': 'sell_signal'}, + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_date': np.datetime64('2018-10-03T00:20:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:25:00.000000000'), + 'open_index': 4, + 'close_index': 4, + 'trade_duration': '', + 'open_rate': 20, + 'close_rate': 10, + 'exit_type': 'sell_signal'}, + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_date': np.datetime64('2018-10-03T00:20:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:25:00.000000000'), + 'open_index': 4, + 'close_index': 4, + 'trade_duration': '', + 'open_rate': 20, + 'close_rate': 10, + 'exit_type': 'sell_signal'}, + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_date': np.datetime64('2018-10-03T00:20:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:25:00.000000000'), + 'open_index': 4, + 'close_index': 4, + 'trade_duration': '', + 'open_rate': 20, + 'close_rate': 10, + 'exit_type': 'sell_signal'}, + + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_date': np.datetime64('2018-10-03T00:30:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:40:00.000000000'), + 'open_index': 6, + 'close_index': 7, + 'trade_duration': '', + 'open_rate': 26, + 'close_rate': 134, + 'exit_type': 'sell_signal'} + ] + + trades_df = DataFrame(trades) + trades_df = edge._fill_calculable_fields(trades_df) + final = edge._process_expectancy(trades_df) + + assert 'TEST/BTC' in final + assert final['TEST/BTC'].stoploss == -0.9 + assert final['TEST/BTC'].nb_trades == len(trades_df) - 1 + assert round(final['TEST/BTC'].winrate, 10) == 0.0 + + +def test_process_expectancy_only_wins(mocker, edge_conf, fee,): + edge_conf['edge']['min_trade_number'] = 2 + freqtrade = get_patched_freqtradebot(mocker, edge_conf) + + freqtrade.exchange.get_fee = fee + edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) + + trades = [ + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_date': np.datetime64('2018-10-03T00:05:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:10:00.000000000'), + 'open_index': 1, + 'close_index': 1, + 'trade_duration': '', + 'open_rate': 15, + 'close_rate': 17, + 'exit_type': 'sell_signal'}, + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_date': np.datetime64('2018-10-03T00:20:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:25:00.000000000'), + 'open_index': 4, + 'close_index': 4, + 'trade_duration': '', + 'open_rate': 10, + 'close_rate': 20, + 'exit_type': 'sell_signal'}, + {'pair': 'TEST/BTC', + 'stoploss': -0.9, + 'profit_percent': '', + 'profit_abs': '', + 'open_date': np.datetime64('2018-10-03T00:30:00.000000000'), + 'close_date': np.datetime64('2018-10-03T00:40:00.000000000'), + 'open_index': 6, + 'close_index': 7, + 'trade_duration': '', + 'open_rate': 26, + 'close_rate': 134, + 'exit_type': 'sell_signal'} + ] + + trades_df = DataFrame(trades) + trades_df = edge._fill_calculable_fields(trades_df) + final = edge._process_expectancy(trades_df) + + assert 'TEST/BTC' in final + assert final['TEST/BTC'].stoploss == -0.9 + assert final['TEST/BTC'].nb_trades == len(trades_df) + assert round(final['TEST/BTC'].winrate, 10) == 1.0 + assert round(final['TEST/BTC'].risk_reward_ratio, 10) == float('inf') + assert round(final['TEST/BTC'].expectancy, 10) == float('inf') diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 52faa284b..f2b508761 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -4,9 +4,9 @@ from unittest.mock import MagicMock import ccxt import pytest -from freqtrade.exceptions import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) +from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException from tests.conftest import get_patched_exchange +from tests.exchange.test_exchange import ccxt_exceptionhandlers @pytest.mark.parametrize('limitratio,expected', [ @@ -62,15 +62,9 @@ def test_stoploss_order_binance(default_conf, mocker, limitratio, expected): exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) - with pytest.raises(TemporaryError): - api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) - - with pytest.raises(OperationalException, match=r".*DeadBeef.*"): - api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, "binance", + "stoploss", "create_order", retries=1, + pair='ETH/BTC', amount=1, stop_price=220, order_types={}) def test_stoploss_order_dry_run_binance(default_conf, mocker): diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 32163f696..a42ff52e4 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1,31 +1,28 @@ -# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement -# pragma pylint: disable=protected-access import copy import logging -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from random import randint -from unittest.mock import MagicMock, Mock, PropertyMock +from unittest.mock import MagicMock, Mock, PropertyMock, patch import arrow import ccxt import pytest from pandas import DataFrame -from freqtrade.exceptions import (DependencyException, InvalidOrderException, +from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException, OperationalException, TemporaryError) -from freqtrade.exchange import Binance, Exchange, Kraken -from freqtrade.exchange.common import API_RETRY_COUNT -from freqtrade.exchange.exchange import (market_is_active, symbol_is_pair, - timeframe_to_minutes, - timeframe_to_msecs, - timeframe_to_next_date, - timeframe_to_prev_date, +from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken +from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, + calculate_backoff) +from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs, + timeframe_to_next_date, timeframe_to_prev_date, timeframe_to_seconds) from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_patched_exchange, log_has, log_has_re + # Make sure to always keep one exchange here which is NOT subclassed!! -EXCHANGES = ['bittrex', 'binance', 'kraken', ] +EXCHANGES = ['bittrex', 'binance', 'kraken', 'ftx'] # Source: https://stackoverflow.com/questions/29881236/how-to-mock-asyncio-coroutines @@ -37,12 +34,20 @@ def get_mock_coro(return_value): def ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, - fun, mock_ccxt_fun, **kwargs): + fun, mock_ccxt_fun, retries=API_RETRY_COUNT + 1, **kwargs): + + with patch('freqtrade.exchange.common.time.sleep'): + with pytest.raises(DDosProtection): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.DDoSProtection("DDos")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == retries + with pytest.raises(TemporaryError): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError("DeaDBeef")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == retries with pytest.raises(OperationalException): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) @@ -51,12 +56,21 @@ def ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, assert api_mock.__dict__[mock_ccxt_fun].call_count == 1 -async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun, **kwargs): +async def async_ccxt_exception(mocker, default_conf, api_mock, fun, mock_ccxt_fun, + retries=API_RETRY_COUNT + 1, **kwargs): + + with patch('freqtrade.exchange.common.asyncio.sleep', get_mock_coro(None)): + with pytest.raises(DDosProtection): + api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.DDoSProtection("Dooh")) + exchange = get_patched_exchange(mocker, default_conf, api_mock) + await getattr(exchange, fun)(**kwargs) + assert api_mock.__dict__[mock_ccxt_fun].call_count == retries + with pytest.raises(TemporaryError): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.NetworkError("DeadBeef")) exchange = get_patched_exchange(mocker, default_conf, api_mock) await getattr(exchange, fun)(**kwargs) - assert api_mock.__dict__[mock_ccxt_fun].call_count == API_RETRY_COUNT + 1 + assert api_mock.__dict__[mock_ccxt_fun].call_count == retries with pytest.raises(OperationalException): api_mock.__dict__[mock_ccxt_fun] = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) @@ -134,11 +148,19 @@ def test_exchange_resolver(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange.validate_pairs') mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') - exchange = ExchangeResolver.load_exchange('Bittrex', default_conf) + + exchange = ExchangeResolver.load_exchange('huobi', default_conf) assert isinstance(exchange, Exchange) assert log_has_re(r"No .* specific subclass found. Using the generic class instead.", caplog) caplog.clear() + exchange = ExchangeResolver.load_exchange('Bittrex', default_conf) + assert isinstance(exchange, Exchange) + assert isinstance(exchange, Bittrex) + assert not log_has_re(r"No .* specific subclass found. Using the generic class instead.", + caplog) + caplog.clear() + exchange = ExchangeResolver.load_exchange('kraken', default_conf) assert isinstance(exchange, Exchange) assert isinstance(exchange, Kraken) @@ -319,7 +341,12 @@ def test_set_sandbox_exception(default_conf, mocker): def test__load_async_markets(default_conf, mocker, caplog): - exchange = get_patched_exchange(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt') + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + exchange = Exchange(default_conf) exchange._api_async.load_markets = get_mock_coro(None) exchange._load_async_markets() assert exchange._api_async.load_markets.call_count == 1 @@ -352,7 +379,7 @@ def test__load_markets(default_conf, mocker, caplog): assert ex.markets == expected_return -def test__reload_markets(default_conf, mocker, caplog): +def test_reload_markets(default_conf, mocker, caplog): caplog.set_level(logging.DEBUG) initial_markets = {'ETH/BTC': {}} @@ -365,23 +392,26 @@ def test__reload_markets(default_conf, mocker, caplog): default_conf['exchange']['markets_refresh_interval'] = 10 exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance", mock_markets=False) - exchange._last_markets_refresh = arrow.utcnow().timestamp + exchange._load_async_markets = MagicMock() + exchange._last_markets_refresh = arrow.utcnow().int_timestamp updated_markets = {'ETH/BTC': {}, "LTC/BTC": {}} assert exchange.markets == initial_markets # less than 10 minutes have passed, no reload - exchange._reload_markets() + exchange.reload_markets() assert exchange.markets == initial_markets + assert exchange._load_async_markets.call_count == 0 # more than 10 minutes have passed, reload is executed - exchange._last_markets_refresh = arrow.utcnow().timestamp - 15 * 60 - exchange._reload_markets() + exchange._last_markets_refresh = arrow.utcnow().int_timestamp - 15 * 60 + exchange.reload_markets() assert exchange.markets == updated_markets + assert exchange._load_async_markets.call_count == 1 assert log_has('Performing scheduled market reload..', caplog) -def test__reload_markets_exception(default_conf, mocker, caplog): +def test_reload_markets_exception(default_conf, mocker, caplog): caplog.set_level(logging.DEBUG) api_mock = MagicMock() @@ -390,7 +420,7 @@ def test__reload_markets_exception(default_conf, mocker, caplog): exchange = get_patched_exchange(mocker, default_conf, api_mock, id="binance") # less than 10 minutes have passed, no reload - exchange._reload_markets() + exchange.reload_markets() assert exchange._last_markets_refresh == 0 assert log_has_re(r"Could not reload markets.*", caplog) @@ -578,7 +608,7 @@ def test_validate_pairs_stakecompatibility_fail(default_conf, mocker, caplog): ('5m'), ("1m"), ("15m"), ("1h") ]) def test_validate_timeframes(default_conf, mocker, timeframe): - default_conf["ticker_interval"] = timeframe + default_conf["timeframe"] = timeframe api_mock = MagicMock() id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock @@ -596,7 +626,7 @@ def test_validate_timeframes(default_conf, mocker, timeframe): def test_validate_timeframes_failed(default_conf, mocker): - default_conf["ticker_interval"] = "3m" + default_conf["timeframe"] = "3m" api_mock = MagicMock() id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock @@ -613,7 +643,7 @@ def test_validate_timeframes_failed(default_conf, mocker): with pytest.raises(OperationalException, match=r"Invalid timeframe '3m'. This exchange supports.*"): Exchange(default_conf) - default_conf["ticker_interval"] = "15s" + default_conf["timeframe"] = "15s" with pytest.raises(OperationalException, match=r"Timeframes < 1m are currently not supported by Freqtrade."): @@ -621,7 +651,7 @@ def test_validate_timeframes_failed(default_conf, mocker): def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker): - default_conf["ticker_interval"] = "3m" + default_conf["timeframe"] = "3m" api_mock = MagicMock() id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock @@ -641,7 +671,7 @@ def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker): def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker): - default_conf["ticker_interval"] = "3m" + default_conf["timeframe"] = "3m" api_mock = MagicMock() id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock @@ -662,7 +692,7 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker): def test_validate_timeframes_not_in_config(default_conf, mocker): - del default_conf["ticker_interval"] + del default_conf["timeframe"] api_mock = MagicMock() id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock @@ -689,13 +719,13 @@ def test_validate_order_types(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') mocker.patch('freqtrade.exchange.Exchange.name', 'Bittrex') + default_conf['order_types'] = { 'buy': 'limit', 'sell': 'limit', 'stoploss': 'market', 'stoploss_on_exchange': False } - Exchange(default_conf) type(api_mock).has = PropertyMock(return_value={'createMarketOrder': False}) @@ -705,9 +735,8 @@ def test_validate_order_types(default_conf, mocker): 'buy': 'limit', 'sell': 'limit', 'stoploss': 'market', - 'stoploss_on_exchange': 'false' + 'stoploss_on_exchange': False } - with pytest.raises(OperationalException, match=r'Exchange .* does not support market orders.'): Exchange(default_conf) @@ -718,7 +747,6 @@ def test_validate_order_types(default_conf, mocker): 'stoploss': 'limit', 'stoploss_on_exchange': True } - with pytest.raises(OperationalException, match=r'On exchange stoploss is not supported for .*'): Exchange(default_conf) @@ -784,7 +812,7 @@ def test_dry_run_order(default_conf, mocker, side, exchange_name): assert f'dry_run_{side}_' in order["id"] assert order["side"] == side assert order["type"] == "limit" - assert order["pair"] == "ETH/BTC" + assert order["symbol"] == "ETH/BTC" @pytest.mark.parametrize("side", [ @@ -1119,9 +1147,10 @@ def test_get_balance_prod(default_conf, mocker, exchange_name): exchange.get_balance(currency='BTC') -def test_get_balances_dry_run(default_conf, mocker): +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_get_balances_dry_run(default_conf, mocker, exchange_name): default_conf['dry_run'] = True - exchange = get_patched_exchange(mocker, default_conf) + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) assert exchange.get_balances() == {} @@ -1243,7 +1272,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) ohlcv = [ [ - arrow.utcnow().timestamp * 1000, # unix timestamp ms + arrow.utcnow().int_timestamp * 1000, # unix timestamp ms 1, # open 2, # high 3, # low @@ -1258,18 +1287,32 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls - since = 5 * 60 * 500 * 1.8 - ret = exchange.get_historic_ohlcv(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000)) + + since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8 + ret = exchange.get_historic_ohlcv(pair, "5m", int(( + arrow.utcnow().int_timestamp - since) * 1000)) assert exchange._async_get_candle_history.call_count == 2 # Returns twice the above OHLCV data assert len(ret) == 2 + caplog.clear() -def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: + async def mock_get_candle_hist_error(pair, *args, **kwargs): + raise TimeoutError() + + exchange._async_get_candle_history = MagicMock(side_effect=mock_get_candle_hist_error) + ret = exchange.get_historic_ohlcv(pair, "5m", int( + (arrow.utcnow().int_timestamp - since) * 1000)) + assert log_has_re(r"Async code raised an exception: .*", caplog) + + +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) ohlcv = [ [ - (arrow.utcnow().timestamp - 1) * 1000, # unix timestamp ms + arrow.utcnow().int_timestamp * 1000, # unix timestamp ms 1, # open 2, # high 3, # low @@ -1277,7 +1320,56 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: 5, # volume (in quote currency) ], [ - arrow.utcnow().timestamp * 1000, # unix timestamp ms + arrow.utcnow().shift(minutes=5).int_timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ], + [ + arrow.utcnow().shift(minutes=10).int_timestamp * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + pair = 'ETH/BTC' + + async def mock_candle_hist(pair, timeframe, since_ms): + return pair, timeframe, ohlcv + + exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) + # one_call calculation * 1.8 should do 2 calls + + since = 5 * 60 * exchange._ft_has['ohlcv_candle_limit'] * 1.8 + ret = exchange.get_historic_ohlcv_as_df(pair, "5m", int(( + arrow.utcnow().int_timestamp - since) * 1000)) + + assert exchange._async_get_candle_history.call_count == 2 + # Returns twice the above OHLCV data + assert len(ret) == 2 + assert isinstance(ret, DataFrame) + assert 'date' in ret.columns + assert 'open' in ret.columns + assert 'close' in ret.columns + assert 'high' in ret.columns + + +def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: + ohlcv = [ + [ + (arrow.utcnow().int_timestamp - 1) * 1000, # unix timestamp ms + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ], + [ + arrow.utcnow().int_timestamp * 1000, # unix timestamp ms 3, # open 1, # high 4, # low @@ -1293,6 +1385,12 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')] # empty dicts assert not exchange._klines + exchange.refresh_latest_ohlcv(pairs, cache=False) + # No caching + assert not exchange._klines + assert exchange._api_async.fetch_ohlcv.call_count == 2 + exchange._api_async.fetch_ohlcv.reset_mock() + exchange.refresh_latest_ohlcv(pairs) assert log_has(f'Refreshing candle (OHLCV) data for {len(pairs)} pairs', caplog) @@ -1323,7 +1421,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_name): ohlcv = [ [ - arrow.utcnow().timestamp * 1000, # unix timestamp ms + arrow.utcnow().int_timestamp * 1000, # unix timestamp ms 1, # open 2, # high 3, # low @@ -1350,7 +1448,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ # exchange = Exchange(default_conf) await async_ccxt_exception(mocker, default_conf, MagicMock(), "_async_get_candle_history", "fetch_ohlcv", - pair='ABCD/BTC', timeframe=default_conf['ticker_interval']) + pair='ABCD/BTC', timeframe=default_conf['timeframe']) api_mock = MagicMock() with pytest.raises(OperationalException, @@ -1358,14 +1456,14 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", - (arrow.utcnow().timestamp - 2000) * 1000) + (arrow.utcnow().int_timestamp - 2000) * 1000) with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching ' r'historical candle \(OHLCV\) data\..*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", - (arrow.utcnow().timestamp - 2000) * 1000) + (arrow.utcnow().int_timestamp - 2000) * 1000) @pytest.mark.asyncio @@ -1407,15 +1505,34 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): assert exchange._klines assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert type(res) is list - assert len(res) == 2 + assert type(res) is dict + assert len(res) == 1 # Test that each is in list at least once as order is not guaranteed - assert type(res[0]) is tuple or type(res[1]) is tuple - assert type(res[0]) is TypeError or type(res[1]) is TypeError assert log_has("Error loading ETH/BTC. Result was [[]].", caplog) assert log_has("Async code raised an exception: TypeError", caplog) +def test_get_next_limit_in_list(): + limit_range = [5, 10, 20, 50, 100, 500, 1000] + assert Exchange.get_next_limit_in_list(1, limit_range) == 5 + assert Exchange.get_next_limit_in_list(5, limit_range) == 5 + assert Exchange.get_next_limit_in_list(6, limit_range) == 10 + assert Exchange.get_next_limit_in_list(9, limit_range) == 10 + assert Exchange.get_next_limit_in_list(10, limit_range) == 10 + assert Exchange.get_next_limit_in_list(11, limit_range) == 20 + assert Exchange.get_next_limit_in_list(19, limit_range) == 20 + assert Exchange.get_next_limit_in_list(21, limit_range) == 50 + assert Exchange.get_next_limit_in_list(51, limit_range) == 100 + assert Exchange.get_next_limit_in_list(1000, limit_range) == 1000 + # Going over the limit ... + assert Exchange.get_next_limit_in_list(1001, limit_range) == 1000 + assert Exchange.get_next_limit_in_list(2000, limit_range) == 1000 + + assert Exchange.get_next_limit_in_list(21, None) == 21 + assert Exchange.get_next_limit_in_list(100, None) == 100 + assert Exchange.get_next_limit_in_list(1000, None) == 1000 + + @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name): default_conf['exchange']['name'] = exchange_name @@ -1428,6 +1545,19 @@ def test_fetch_l2_order_book(default_conf, mocker, order_book_l2, exchange_name) assert 'asks' in order_book assert len(order_book['bids']) == 10 assert len(order_book['asks']) == 10 + assert api_mock.fetch_l2_order_book.call_args_list[0][0][0] == 'ETH/BTC' + + for val in [1, 5, 10, 12, 20, 50, 100]: + api_mock.fetch_l2_order_book.reset_mock() + + order_book = exchange.fetch_l2_order_book(pair='ETH/BTC', limit=val) + assert api_mock.fetch_l2_order_book.call_args_list[0][0][0] == 'ETH/BTC' + # Not all exchanges support all limits for orderbook + if not exchange._ft_has['l2_limit_range'] or val in exchange._ft_has['l2_limit_range']: + assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == val + else: + next_limit = exchange.get_next_limit_in_list(val, exchange._ft_has['l2_limit_range']) + assert api_mock.fetch_l2_order_book.call_args_list[0][0][1] == next_limit @pytest.mark.parametrize("exchange_name", EXCHANGES) @@ -1480,7 +1610,7 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) sort_mock = mocker.patch('freqtrade.exchange.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the OHLCV data sort - res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['timeframe']) assert res[0] == 'ETH/BTC' res_ohlcv = res[2] @@ -1517,9 +1647,9 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na # Reset sort mock sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the OHLCV data sort - res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) + res = await exchange._async_get_candle_history('ETH/BTC', default_conf['timeframe']) assert res[0] == 'ETH/BTC' - assert res[1] == default_conf['ticker_interval'] + assert res[1] == default_conf['timeframe'] res_ohlcv = res[2] # Sorted not called again - data is already in order assert sort_mock.call_count == 0 @@ -1577,13 +1707,13 @@ async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, with pytest.raises(OperationalException, match=r'Could not fetch trade data*'): api_mock.fetch_trades = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000) + await exchange._async_fetch_trades(pair, since=(arrow.utcnow().int_timestamp - 2000) * 1000) with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching ' r'historical trade data\..*'): api_mock.fetch_trades = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_fetch_trades(pair, since=(arrow.utcnow().timestamp - 2000) * 1000) + await exchange._async_fetch_trades(pair, since=(arrow.utcnow().int_timestamp - 2000) * 1000) @pytest.mark.asyncio @@ -1733,6 +1863,15 @@ def test_cancel_order_dry_run(default_conf, mocker, exchange_name): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {} + assert exchange.cancel_stoploss_order(order_id='123', pair='TKN/BTC') == {} + + order = exchange.buy('ETH/BTC', 'limit', 5, 0.55, 'gtc') + + cancel_order = exchange.cancel_order(order_id=order['id'], pair='ETH/BTC') + assert order['id'] == cancel_order['id'] + assert order['amount'] == cancel_order['amount'] + assert order['symbol'] == cancel_order['symbol'] + assert cancel_order['status'] == 'canceled' @pytest.mark.parametrize("exchange_name", EXCHANGES) @@ -1792,7 +1931,7 @@ def test_cancel_order_with_result_error(default_conf, mocker, exchange_name, cap res = exchange.cancel_order_with_result('1234', 'ETH/BTC', 1541) assert isinstance(res, dict) - assert log_has("Could not cancel order 1234.", caplog) + assert log_has("Could not cancel order 1234 for ETH/BTC.", caplog) assert log_has("Could not fetch cancelled order 1234.", caplog) assert res['amount'] == 1541 @@ -1818,34 +1957,125 @@ def test_cancel_order(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_order(default_conf, mocker, exchange_name): +def test_cancel_stoploss_order(default_conf, mocker, exchange_name): + default_conf['dry_run'] = False + api_mock = MagicMock() + api_mock.cancel_order = MagicMock(return_value=123) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + assert exchange.cancel_stoploss_order(order_id='_', pair='TKN/BTC') == 123 + + with pytest.raises(InvalidOrderException): + api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + exchange.cancel_stoploss_order(order_id='_', pair='TKN/BTC') + assert api_mock.cancel_order.call_count == 1 + + ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, + "cancel_stoploss_order", "cancel_order", + order_id='_', pair='TKN/BTC') + + +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_fetch_order(default_conf, mocker, exchange_name): default_conf['dry_run'] = True order = MagicMock() order.myid = 123 exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) exchange._dry_run_open_orders['X'] = order - assert exchange.get_order('X', 'TKN/BTC').myid == 123 + assert exchange.fetch_order('X', 'TKN/BTC').myid == 123 with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'): - exchange.get_order('Y', 'TKN/BTC') + exchange.fetch_order('Y', 'TKN/BTC') default_conf['dry_run'] = False api_mock = MagicMock() api_mock.fetch_order = MagicMock(return_value=456) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - assert exchange.get_order('X', 'TKN/BTC') == 456 + assert exchange.fetch_order('X', 'TKN/BTC') == 456 with pytest.raises(InvalidOrderException): api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.get_order(order_id='_', pair='TKN/BTC') + exchange.fetch_order(order_id='_', pair='TKN/BTC') + assert api_mock.fetch_order.call_count == 1 + + api_mock.fetch_order = MagicMock(side_effect=ccxt.OrderNotFound("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + with patch('freqtrade.exchange.common.time.sleep') as tm: + with pytest.raises(InvalidOrderException): + exchange.fetch_order(order_id='_', pair='TKN/BTC') + # Ensure backoff is called + assert tm.call_args_list[0][0][0] == 1 + assert tm.call_args_list[1][0][0] == 2 + if API_FETCH_ORDER_RETRY_COUNT > 2: + assert tm.call_args_list[2][0][0] == 5 + if API_FETCH_ORDER_RETRY_COUNT > 3: + assert tm.call_args_list[3][0][0] == 10 + assert api_mock.fetch_order.call_count == API_FETCH_ORDER_RETRY_COUNT + 1 + + ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, + 'fetch_order', 'fetch_order', retries=API_FETCH_ORDER_RETRY_COUNT + 1, + order_id='_', pair='TKN/BTC') + + +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_fetch_stoploss_order(default_conf, mocker, exchange_name): + # Don't test FTX here - that needs a seperate test + if exchange_name == 'ftx': + return + default_conf['dry_run'] = True + order = MagicMock() + order.myid = 123 + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + exchange._dry_run_open_orders['X'] = order + assert exchange.fetch_stoploss_order('X', 'TKN/BTC').myid == 123 + + with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'): + exchange.fetch_stoploss_order('Y', 'TKN/BTC') + + default_conf['dry_run'] = False + api_mock = MagicMock() + api_mock.fetch_order = MagicMock(return_value=456) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + assert exchange.fetch_stoploss_order('X', 'TKN/BTC') == 456 + + with pytest.raises(InvalidOrderException): + api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + exchange.fetch_stoploss_order(order_id='_', pair='TKN/BTC') assert api_mock.fetch_order.call_count == 1 ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, - 'get_order', 'fetch_order', + 'fetch_stoploss_order', 'fetch_order', + retries=API_FETCH_ORDER_RETRY_COUNT + 1, order_id='_', pair='TKN/BTC') +def test_fetch_order_or_stoploss_order(default_conf, mocker): + exchange = get_patched_exchange(mocker, default_conf, id='binance') + fetch_order_mock = MagicMock() + fetch_stoploss_order_mock = MagicMock() + mocker.patch.multiple('freqtrade.exchange.Exchange', + fetch_order=fetch_order_mock, + fetch_stoploss_order=fetch_stoploss_order_mock, + ) + + exchange.fetch_order_or_stoploss_order('1234', 'ETH/BTC', False) + assert fetch_order_mock.call_count == 1 + assert fetch_order_mock.call_args_list[0][0][0] == '1234' + assert fetch_order_mock.call_args_list[0][0][1] == 'ETH/BTC' + assert fetch_stoploss_order_mock.call_count == 0 + + fetch_order_mock.reset_mock() + fetch_stoploss_order_mock.reset_mock() + + exchange.fetch_order_or_stoploss_order('1234', 'ETH/BTC', True) + assert fetch_order_mock.call_count == 0 + assert fetch_stoploss_order_mock.call_count == 1 + assert fetch_stoploss_order_mock.call_args_list[0][0][0] == '1234' + assert fetch_stoploss_order_mock.call_args_list[0][0][1] == 'ETH/BTC' + + @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_name(default_conf, mocker, exchange_name): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) @@ -1923,7 +2153,7 @@ def test_get_fee(default_conf, mocker, exchange_name): def test_stoploss_order_unsupported_exchange(default_conf, mocker): - exchange = get_patched_exchange(mocker, default_conf, 'bittrex') + exchange = get_patched_exchange(mocker, default_conf, id='bittrex') with pytest.raises(OperationalException, match=r"stoploss is not implemented .*"): exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) @@ -2050,6 +2280,13 @@ def test_get_markets(default_conf, mocker, markets, assert sorted(pairs.keys()) == sorted(expected_keys) +def test_get_markets_error(default_conf, mocker): + ex = get_patched_exchange(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=None)) + with pytest.raises(OperationalException, match="Markets were not loaded."): + ex.get_markets('LTC', 'USDT', True, False) + + def test_timeframe_to_minutes(): assert timeframe_to_minutes("5m") == 5 assert timeframe_to_minutes("10m") == 10 @@ -2120,26 +2357,46 @@ def test_timeframe_to_next_date(): date = datetime.now(tz=timezone.utc) assert timeframe_to_next_date("5m") > date + date = datetime(2019, 8, 12, 13, 30, 0, tzinfo=timezone.utc) + assert timeframe_to_next_date("5m", date) == date + timedelta(minutes=5) -@pytest.mark.parametrize("market_symbol,base_currency,quote_currency,expected_result", [ - ("BTC/USDT", None, None, True), - ("USDT/BTC", None, None, True), - ("BTCUSDT", None, None, False), - ("BTC/USDT", None, "USDT", True), - ("USDT/BTC", None, "USDT", False), - ("BTCUSDT", None, "USDT", False), - ("BTC/USDT", "BTC", None, True), - ("USDT/BTC", "BTC", None, False), - ("BTCUSDT", "BTC", None, False), - ("BTC/USDT", "BTC", "USDT", True), - ("BTC/USDT", "USDT", "BTC", False), - ("BTC/USDT", "BTC", "USD", False), - ("BTCUSDT", "BTC", "USDT", False), - ("BTC/", None, None, False), - ("/USDT", None, None, False), + +@pytest.mark.parametrize("market_symbol,base,quote,exchange,add_dict,expected_result", [ + ("BTC/USDT", 'BTC', 'USDT', "binance", {}, True), + ("USDT/BTC", 'USDT', 'BTC', "binance", {}, True), + ("USDT/BTC", 'BTC', 'USDT', "binance", {}, False), # Reversed currencies + ("BTCUSDT", 'BTC', 'USDT', "binance", {}, False), # No seperating / + ("BTCUSDT", None, "USDT", "binance", {}, False), # + ("USDT/BTC", "BTC", None, "binance", {}, False), + ("BTCUSDT", "BTC", None, "binance", {}, False), + ("BTC/USDT", "BTC", "USDT", "binance", {}, True), + ("BTC/USDT", "USDT", "BTC", "binance", {}, False), # reversed currencies + ("BTC/USDT", "BTC", "USD", "binance", {}, False), # Wrong quote currency + ("BTC/", "BTC", 'UNK', "binance", {}, False), + ("/USDT", 'UNK', 'USDT', "binance", {}, False), + ("BTC/EUR", 'BTC', 'EUR', "kraken", {"darkpool": False}, True), + ("EUR/BTC", 'EUR', 'BTC', "kraken", {"darkpool": False}, True), + ("EUR/BTC", 'BTC', 'EUR', "kraken", {"darkpool": False}, False), # Reversed currencies + ("BTC/EUR", 'BTC', 'USD', "kraken", {"darkpool": False}, False), # wrong quote currency + ("BTC/EUR", 'BTC', 'EUR', "kraken", {"darkpool": True}, False), # no darkpools + ("BTC/EUR.d", 'BTC', 'EUR', "kraken", {"darkpool": True}, False), # no darkpools + ("BTC/USD", 'BTC', 'USD', "ftx", {'spot': True}, True), + ("USD/BTC", 'USD', 'BTC', "ftx", {'spot': True}, True), + ("BTC/USD", 'BTC', 'USDT', "ftx", {'spot': True}, False), # Wrong quote currency + ("BTC/USD", 'USD', 'BTC', "ftx", {'spot': True}, False), # Reversed currencies + ("BTC/USD", 'BTC', 'USD', "ftx", {'spot': False}, False), # Can only trade spot markets + ("BTC-PERP", 'BTC', 'USD', "ftx", {'spot': False}, False), # Can only trade spot markets ]) -def test_symbol_is_pair(market_symbol, base_currency, quote_currency, expected_result) -> None: - assert symbol_is_pair(market_symbol, base_currency, quote_currency) == expected_result +def test_market_is_tradable(mocker, default_conf, market_symbol, base, + quote, add_dict, exchange, expected_result) -> None: + ex = get_patched_exchange(mocker, default_conf, id=exchange) + market = { + 'symbol': market_symbol, + 'base': base, + 'quote': quote, + **(add_dict), + } + assert ex.market_is_tradable(market) == expected_result @pytest.mark.parametrize("market,expected_result", [ @@ -2210,3 +2467,27 @@ def test_calculate_fee_rate(mocker, default_conf, order, expected) -> None: ex = get_patched_exchange(mocker, default_conf) assert ex.calculate_fee_rate(order) == expected + + +@pytest.mark.parametrize('retrycount,max_retries,expected', [ + (0, 3, 10), + (1, 3, 5), + (2, 3, 2), + (3, 3, 1), + (0, 1, 2), + (1, 1, 1), + (0, 4, 17), + (1, 4, 10), + (2, 4, 5), + (3, 4, 2), + (4, 4, 1), + (0, 5, 26), + (1, 5, 17), + (2, 5, 10), + (3, 5, 5), + (4, 5, 2), + (5, 5, 1), + +]) +def test_calculate_backoff(retrycount, max_retries, expected): + assert calculate_backoff(retrycount, max_retries) == expected diff --git a/tests/exchange/test_ftx.py b/tests/exchange/test_ftx.py new file mode 100644 index 000000000..17cfb26fa --- /dev/null +++ b/tests/exchange/test_ftx.py @@ -0,0 +1,158 @@ +from random import randint +from unittest.mock import MagicMock + +import ccxt +import pytest + +from freqtrade.exceptions import DependencyException, InvalidOrderException +from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT +from tests.conftest import get_patched_exchange + +from .test_exchange import ccxt_exceptionhandlers + + +STOPLOSS_ORDERTYPE = 'stop' + + +def test_stoploss_order_ftx(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + + default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'ftx') + + # stoploss_on_exchange_limit_ratio is irrelevant for ftx market orders + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=190, + order_types={'stoploss_on_exchange_limit_ratio': 1.05}) + + assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC' + assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_ORDERTYPE + assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' + assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 + assert api_mock.create_order.call_args_list[0][1]['price'] == 190 + assert 'orderPrice' not in api_mock.create_order.call_args_list[0][1]['params'] + + assert api_mock.create_order.call_count == 1 + + api_mock.create_order.reset_mock() + + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC' + assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_ORDERTYPE + assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' + assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 + assert api_mock.create_order.call_args_list[0][1]['price'] == 220 + assert 'orderPrice' not in api_mock.create_order.call_args_list[0][1]['params'] + + api_mock.create_order.reset_mock() + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, + order_types={'stoploss': 'limit'}) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC' + assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_ORDERTYPE + assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' + assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 + assert api_mock.create_order.call_args_list[0][1]['price'] == 220 + assert 'orderPrice' in api_mock.create_order.call_args_list[0][1]['params'] + assert api_mock.create_order.call_args_list[0][1]['params']['orderPrice'] == 217.8 + + # test exception handling + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'ftx') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + with pytest.raises(InvalidOrderException): + api_mock.create_order = MagicMock( + side_effect=ccxt.InvalidOrder("ftx Order would trigger immediately.")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'ftx') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + ccxt_exceptionhandlers(mocker, default_conf, api_mock, "ftx", + "stoploss", "create_order", retries=1, + pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + +def test_stoploss_order_dry_run_ftx(default_conf, mocker): + api_mock = MagicMock() + default_conf['dry_run'] = True + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'ftx') + + api_mock.create_order.reset_mock() + + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + assert 'id' in order + assert 'info' in order + assert 'type' in order + + assert order['type'] == STOPLOSS_ORDERTYPE + assert order['price'] == 220 + assert order['amount'] == 1 + + +def test_stoploss_adjust_ftx(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf, id='ftx') + order = { + 'type': STOPLOSS_ORDERTYPE, + 'price': 1500, + } + assert exchange.stoploss_adjust(1501, order) + assert not exchange.stoploss_adjust(1499, order) + # Test with invalid order case ... + order['type'] = 'stop_loss_limit' + assert not exchange.stoploss_adjust(1501, order) + + +def test_fetch_stoploss_order(default_conf, mocker): + default_conf['dry_run'] = True + order = MagicMock() + order.myid = 123 + exchange = get_patched_exchange(mocker, default_conf, id='ftx') + exchange._dry_run_open_orders['X'] = order + assert exchange.fetch_stoploss_order('X', 'TKN/BTC').myid == 123 + + with pytest.raises(InvalidOrderException, match=r'Tried to get an invalid dry-run-order.*'): + exchange.fetch_stoploss_order('Y', 'TKN/BTC') + + default_conf['dry_run'] = False + api_mock = MagicMock() + api_mock.fetch_orders = MagicMock(return_value=[{'id': 'X', 'status': '456'}]) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='ftx') + assert exchange.fetch_stoploss_order('X', 'TKN/BTC')['status'] == '456' + + api_mock.fetch_orders = MagicMock(return_value=[{'id': 'Y', 'status': '456'}]) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='ftx') + with pytest.raises(InvalidOrderException, match=r"Could not get stoploss order for id X"): + exchange.fetch_stoploss_order('X', 'TKN/BTC')['status'] + + with pytest.raises(InvalidOrderException): + api_mock.fetch_orders = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id='ftx') + exchange.fetch_stoploss_order(order_id='_', pair='TKN/BTC') + assert api_mock.fetch_orders.call_count == 1 + + ccxt_exceptionhandlers(mocker, default_conf, api_mock, 'ftx', + 'fetch_stoploss_order', 'fetch_orders', + retries=API_FETCH_ORDER_RETRY_COUNT + 1, + order_id='_', pair='TKN/BTC') diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index d63dd66cc..3803658eb 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -1,17 +1,18 @@ -# pragma pylint: disable=missing-docstring, C0103, bad-continuation, global-statement -# pragma pylint: disable=protected-access from random import randint from unittest.mock import MagicMock import ccxt import pytest -from freqtrade.exceptions import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) +from freqtrade.exceptions import DependencyException, InvalidOrderException from tests.conftest import get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers +STOPLOSS_ORDERTYPE = 'stop-loss' +STOPLOSS_LIMIT_ORDERTYPE = 'stop-loss-limit' + + def test_buy_kraken_trading_agreement(default_conf, mocker): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) @@ -156,10 +157,10 @@ def test_get_balances_prod(default_conf, mocker): "get_balances", "fetch_balance") -def test_stoploss_order_kraken(default_conf, mocker): +@pytest.mark.parametrize('ordertype', ['market', 'limit']) +def test_stoploss_order_kraken(default_conf, mocker, ordertype): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) - order_type = 'stop-loss' api_mock.create_order = MagicMock(return_value={ 'id': order_id, @@ -174,24 +175,26 @@ def test_stoploss_order_kraken(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') - # stoploss_on_exchange_limit_ratio is irrelevant for kraken market orders - order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=190, - order_types={'stoploss_on_exchange_limit_ratio': 1.05}) - assert api_mock.create_order.call_count == 1 - - api_mock.create_order.reset_mock() - - order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, + order_types={'stoploss': ordertype, + 'stoploss_on_exchange_limit_ratio': 0.99 + }) assert 'id' in order assert 'info' in order assert order['id'] == order_id assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC' - assert api_mock.create_order.call_args_list[0][1]['type'] == order_type + if ordertype == 'limit': + assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_LIMIT_ORDERTYPE + assert api_mock.create_order.call_args_list[0][1]['params'] == { + 'trading_agreement': 'agree', 'price2': 217.8} + else: + assert api_mock.create_order.call_args_list[0][1]['type'] == STOPLOSS_ORDERTYPE + assert api_mock.create_order.call_args_list[0][1]['params'] == { + 'trading_agreement': 'agree'} assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 assert api_mock.create_order.call_args_list[0][1]['price'] == 220 - assert api_mock.create_order.call_args_list[0][1]['params'] == {'trading_agreement': 'agree'} # test exception handling with pytest.raises(DependencyException): @@ -205,20 +208,13 @@ def test_stoploss_order_kraken(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) - with pytest.raises(TemporaryError): - api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') - exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) - - with pytest.raises(OperationalException, match=r".*DeadBeef.*"): - api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) - exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') - exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + ccxt_exceptionhandlers(mocker, default_conf, api_mock, "kraken", + "stoploss", "create_order", retries=1, + pair='ETH/BTC', amount=1, stop_price=220, order_types={}) def test_stoploss_order_dry_run_kraken(default_conf, mocker): api_mock = MagicMock() - order_type = 'stop-loss' default_conf['dry_run'] = True mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) @@ -233,7 +229,7 @@ def test_stoploss_order_dry_run_kraken(default_conf, mocker): assert 'info' in order assert 'type' in order - assert order['type'] == order_type + assert order['type'] == STOPLOSS_ORDERTYPE assert order['price'] == 220 assert order['amount'] == 1 @@ -241,7 +237,7 @@ def test_stoploss_order_dry_run_kraken(default_conf, mocker): def test_stoploss_adjust_kraken(mocker, default_conf): exchange = get_patched_exchange(mocker, default_conf, id='kraken') order = { - 'type': 'stop-loss', + 'type': STOPLOSS_ORDERTYPE, 'price': 1500, } assert exchange.stoploss_adjust(1501, order) diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index 8bc66f02c..306850ff6 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -6,6 +6,7 @@ from pandas import DataFrame from freqtrade.exchange import timeframe_to_minutes from freqtrade.strategy.interface import SellType + tests_start_time = arrow.get(2018, 10, 3) tests_timeframe = '1h' diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py new file mode 100644 index 000000000..f06b0ecd3 --- /dev/null +++ b/tests/optimize/conftest.py @@ -0,0 +1,51 @@ +from copy import deepcopy +from datetime import datetime +from pathlib import Path + +import pandas as pd +import pytest + +from freqtrade.optimize.hyperopt import Hyperopt +from freqtrade.strategy.interface import SellType +from tests.conftest import patch_exchange + + +@pytest.fixture(scope='function') +def hyperopt_conf(default_conf): + hyperconf = deepcopy(default_conf) + hyperconf.update({ + 'hyperopt': 'DefaultHyperOpt', + 'hyperopt_loss': 'ShortTradeDurHyperOptLoss', + 'hyperopt_path': str(Path(__file__).parent / 'hyperopts'), + 'epochs': 1, + 'timerange': None, + 'spaces': ['default'], + 'hyperopt_jobs': 1, + }) + return hyperconf + + +@pytest.fixture(scope='function') +def hyperopt(hyperopt_conf, mocker): + + patch_exchange(mocker) + return Hyperopt(hyperopt_conf) + + +@pytest.fixture(scope='function') +def hyperopt_results(): + return pd.DataFrame( + { + 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'profit_percent': [-0.1, 0.2, 0.3], + 'profit_abs': [-0.2, 0.4, 0.6], + 'trade_duration': [10, 30, 10], + 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI], + 'close_date': + [ + datetime(2019, 1, 1, 9, 26, 3, 478039), + datetime(2019, 2, 1, 9, 26, 3, 478039), + datetime(2019, 3, 1, 9, 26, 3, 478039) + ] + } + ) diff --git a/freqtrade/optimize/default_hyperopt.py b/tests/optimize/hyperopts/default_hyperopt.py similarity index 100% rename from freqtrade/optimize/default_hyperopt.py rename to tests/optimize/hyperopts/default_hyperopt.py diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index e7bc76c1d..720ed8c13 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -11,6 +11,7 @@ from tests.conftest import patch_exchange from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset, tests_timeframe) + # Test 0: Sell with signal sell in candle 3 # Test with Stop-loss at 1% tc0 = BTContainer(data=[ @@ -327,6 +328,118 @@ tc20 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] ) +# Test 21: trailing_stop ROI collision. +# Roi should trigger before Trailing stop - otherwise Trailing stop profits can be > ROI +# which cannot happen in reality +# stop-loss: 10%, ROI: 4%, Trailing stop adjusted at the sell candle +tc21 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 4650, 5100, 6172, 0, 0], + [3, 4850, 5050, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.04}, profit_perc=0.04, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)] +) + +# Test 22: trailing_stop Raises in candle 2 - but ROI applies at the same time. +# applying a positive trailing stop of 3% - ROI should apply before trailing stop. +# stop-loss: 10%, ROI: 4%, stoploss adjusted candle 2 +tc22 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 5100, 5100, 6172, 0, 0], + [3, 4850, 5050, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.04}, profit_perc=0.04, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)] +) + +# Test 23: trailing_stop Raises in candle 2 (does not trigger) +# applying a positive trailing stop of 3% since stop_positive_offset is reached. +# ROI is changed after this to 4%, dropping ROI below trailing_stop_positive, causing a sell +# in the candle after the raised stoploss candle with ROI reason. +# Stoploss would trigger in this candle too, but it's no longer relevant. +# stop-loss: 10%, ROI: 4%, stoploss adjusted candle 2, ROI adjusted in candle 3 (causing the sell) +tc23 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 5100, 5100, 6172, 0, 0], + [3, 4850, 5251, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.1, "119": 0.03}, profit_perc=0.03, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] +) + +# Test 24: Sell with signal sell in candle 3 (stoploss also triggers on this candle) +# Stoploss at 1%. +# Stoploss wins over Sell-signal (because sell-signal is acted on in the next candle) +tc24 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4600, 6172, 0, 0], + [3, 5010, 5000, 4855, 5010, 6172, 0, 1], # Triggers stoploss + sellsignal + [4, 5010, 4987, 4977, 4995, 6172, 0, 0], + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], + stop_loss=-0.01, roi={"0": 1}, profit_perc=-0.01, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=3)] +) + +# Test 25: Sell with signal sell in candle 3 (stoploss also triggers on this candle) +# Stoploss at 1%. +# Sell-signal wins over stoploss +tc25 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4600, 6172, 0, 0], + [3, 5010, 5000, 4986, 5010, 6172, 0, 1], + [4, 5010, 4987, 4855, 4995, 6172, 0, 0], # Triggers stoploss + sellsignal acted on + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], + stop_loss=-0.01, roi={"0": 1}, profit_perc=0.002, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=4)] +) + +# Test 26: Sell with signal sell in candle 3 (ROI at signal candle) +# Stoploss at 10% (irrelevant), ROI at 5% (will trigger) +# Sell-signal wins over stoploss +tc26 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4600, 6172, 0, 0], + [3, 5010, 5251, 4986, 5010, 6172, 0, 1], # Triggers ROI, sell-signal + [4, 5010, 4987, 4855, 4995, 6172, 0, 0], + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.05}, profit_perc=0.05, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] +) + +# Test 27: Sell with signal sell in candle 3 (ROI at signal candle) +# Stoploss at 10% (irrelevant), ROI at 5% (will trigger) - Wins over Sell-signal +# TODO: figure out if sell-signal should win over ROI +# Sell-signal wins over stoploss +tc27 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4600, 6172, 0, 0], + [3, 5010, 5012, 4986, 5010, 6172, 0, 1], # sell-signal + [4, 5010, 5251, 4855, 4995, 6172, 0, 0], # Triggers ROI, sell-signal acted on + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.05}, profit_perc=0.05, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=4)] +) TESTS = [ tc0, @@ -350,6 +463,13 @@ TESTS = [ tc18, tc19, tc20, + tc21, + tc22, + tc23, + tc24, + tc25, + tc26, + tc27, ] @@ -360,7 +480,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: """ default_conf["stoploss"] = data.stop_loss default_conf["minimal_roi"] = data.roi - default_conf["ticker_interval"] = tests_timeframe + default_conf["timeframe"] = tests_timeframe default_conf["trailing_stop"] = data.trailing_stop default_conf["trailing_only_offset_is_reached"] = data.trailing_only_offset_is_reached # Only add this to configuration If it's necessary @@ -395,5 +515,5 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: for c, trade in enumerate(data.trades): res = results.iloc[c] assert res.sell_reason == trade.sell_reason - assert res.open_time == _get_frame_time_from_offset(trade.open_tick) - assert res.close_time == _get_frame_time_from_offset(trade.close_tick) + assert res.open_date == _get_frame_time_from_offset(trade.open_tick) + assert res.close_date == _get_frame_time_from_offset(trade.close_tick) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 1c9810ec9..376390664 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -10,11 +10,10 @@ import pytest from arrow import Arrow from freqtrade import constants -from freqtrade.commands.optimize_commands import (setup_optimize_configuration, - start_backtesting) +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_backtesting from freqtrade.configuration import TimeRange from freqtrade.data import history -from freqtrade.data.btanalysis import evaluate_result_multi +from freqtrade.data.btanalysis import BT_DATA_COLUMNS, evaluate_result_multi from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange @@ -26,6 +25,7 @@ from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) + ORDER_TYPES = [ { 'buy': 'limit', @@ -79,9 +79,9 @@ def load_data_test(what, testdatadir): fill_missing=True)} -def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: +def simple_backtest(config, contour, mocker, testdatadir) -> None: patch_exchange(mocker) - config['ticker_interval'] = '1m' + config['timeframe'] = '1m' backtesting = Backtesting(config) data = load_data_test(contour, testdatadir) @@ -95,9 +95,10 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: end_date=max_date, max_open_trades=1, position_stacking=False, + enable_protections=config.get('enable_protections', False), ) # results :: - assert len(results) == num_results + return results # FIX: fixturize this? @@ -165,7 +166,7 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has('Using data directory: {} ...'.format(config['datadir']), caplog) - assert 'ticker_interval' in config + assert 'timeframe' in config assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog) assert 'position_stacking' not in config @@ -189,7 +190,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--config', 'config.json', '--strategy', 'DefaultStrategy', '--datadir', '/foo/bar', - '--ticker-interval', '1m', + '--timeframe', '1m', '--enable-position-stacking', '--disable-max-market-positions', '--timerange', ':100', @@ -208,8 +209,8 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert config['runmode'] == RunMode.BACKTEST assert log_has('Using data directory: {} ...'.format(config['datadir']), caplog) - assert 'ticker_interval' in config - assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', + assert 'timeframe' in config + assert log_has('Parameter -i/--timeframe detected ... Using timeframe: 1m ...', caplog) assert 'position_stacking' in config @@ -286,9 +287,9 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None: assert not backtesting.strategy.order_types["stoploss_on_exchange"] -def test_backtesting_init_no_ticker_interval(mocker, default_conf, caplog) -> None: +def test_backtesting_init_no_timeframe(mocker, default_conf, caplog) -> None: patch_exchange(mocker) - del default_conf['ticker_interval'] + del default_conf['timeframe'] default_conf['strategy_list'] = ['DefaultStrategy', 'SampleStrategy'] @@ -308,6 +309,11 @@ def test_data_with_fee(default_conf, mocker, testdatadir) -> None: assert backtesting.fee == 0.1234 assert fee_mock.call_count == 0 + default_conf['fee'] = 0.0 + backtesting = Backtesting(default_conf) + assert backtesting.fee == 0.0 + assert fee_mock.call_count == 0 + def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) @@ -335,10 +341,10 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats') mocker.patch('freqtrade.optimize.backtesting.show_backtest_results') - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) - default_conf['ticker_interval'] = '1m' + default_conf['timeframe'] = '1m' default_conf['datadir'] = testdatadir default_conf['export'] = None default_conf['timerange'] = '-1510694220' @@ -349,11 +355,12 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: exists = [ 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' - 'up to 2017-11-14T22:59:00+00:00 (0 days)..' + 'Backtesting with data from 2017-11-14 21:17:00 ' + 'up to 2017-11-14 22:59:00 (0 days)..' ] for line in exists: assert log_has(line, caplog) + assert backtesting.strategy.dp._pairlists is not None def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> None: @@ -365,10 +372,10 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> mocker.patch('freqtrade.data.history.get_timerange', get_timerange) patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) - default_conf['ticker_interval'] = "1m" + default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir default_conf['export'] = None default_conf['timerange'] = '20180101-20180102' @@ -385,10 +392,10 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> mocker.patch('freqtrade.data.history.get_timerange', get_timerange) patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=[])) - default_conf['ticker_interval'] = "1m" + default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir default_conf['export'] = None default_conf['timerange'] = '20180101-20180102' @@ -401,6 +408,43 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> Backtesting(default_conf) +def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, tickers) -> None: + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + mocker.patch('freqtrade.exchange.Exchange.get_tickers', tickers) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + patch_exchange(mocker) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['XRP/BTC'])) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.refresh_pairlist') + + default_conf['ticker_interval'] = "1m" + default_conf['datadir'] = testdatadir + default_conf['export'] = None + # Use stoploss from strategy + del default_conf['stoploss'] + default_conf['timerange'] = '20180101-20180102' + + default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] + with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting.'): + Backtesting(default_conf) + + default_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}] + with pytest.raises(OperationalException, + match='PerformanceFilter not allowed for backtesting.'): + Backtesting(default_conf) + + default_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}, ] + Backtesting(default_conf) + + # Multiple strategies + default_conf['strategy_list'] = ['DefaultStrategy', 'TestStrategyLegacy'] + with pytest.raises(OperationalException, + match='PrecisionFilter not allowed for backtesting multiple strategies.'): + Backtesting(default_conf) + + def test_backtest(default_conf, fee, mocker, testdatadir) -> None: default_conf['ask_strategy']['use_sell_signal'] = False mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) @@ -427,34 +471,35 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None: {'pair': [pair, pair], 'profit_percent': [0.0, 0.0], 'profit_abs': [0.0, 0.0], - 'open_time': pd.to_datetime([Arrow(2018, 1, 29, 18, 40, 0).datetime, + 'open_date': pd.to_datetime([Arrow(2018, 1, 29, 18, 40, 0).datetime, Arrow(2018, 1, 30, 3, 30, 0).datetime], utc=True ), - 'close_time': pd.to_datetime([Arrow(2018, 1, 29, 22, 35, 0).datetime, + 'open_rate': [0.104445, 0.10302485], + 'open_fee': [0.0025, 0.0025], + 'close_date': pd.to_datetime([Arrow(2018, 1, 29, 22, 35, 0).datetime, Arrow(2018, 1, 30, 4, 10, 0).datetime], utc=True), - 'open_index': [78, 184], - 'close_index': [125, 192], + 'close_rate': [0.104969, 0.103541], + 'close_fee': [0.0025, 0.0025], + 'amount': [0.00957442, 0.0097064], 'trade_duration': [235, 40], 'open_at_end': [False, False], - 'open_rate': [0.104445, 0.10302485], - 'close_rate': [0.104969, 0.103541], 'sell_reason': [SellType.ROI, SellType.ROI] }) pd.testing.assert_frame_equal(results, expected) data_pair = processed[pair] for _, t in results.iterrows(): - ln = data_pair.loc[data_pair["date"] == t["open_time"]] + ln = data_pair.loc[data_pair["date"] == t["open_date"]] # Check open trade rate alignes to open rate assert ln is not None assert round(ln.iloc[0]["open"], 6) == round(t["open_rate"], 6) # check close trade rate alignes to close rate or is between high and low - ln = data_pair.loc[data_pair["date"] == t["close_time"]] + ln = data_pair.loc[data_pair["date"] == t["close_date"]] assert (round(ln.iloc[0]["open"], 6) == round(t["close_rate"], 6) or round(ln.iloc[0]["low"], 6) < round( t["close_rate"], 6) < round(ln.iloc[0]["high"], 6)) -def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) -> None: +def test_backtest_1min_timeframe(default_conf, fee, mocker, testdatadir) -> None: default_conf['ask_strategy']['use_sell_signal'] = False mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) patch_exchange(mocker) @@ -492,13 +537,52 @@ def test_processed(default_conf, mocker, testdatadir) -> None: assert col in cols -def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir) -> None: - # TODO: Evaluate usefullness of this, the patterns and buy-signls are unrealistic - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - tests = [['raise', 19], ['lower', 0], ['sine', 35]] +def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatadir) -> None: + # While this test IS a copy of test_backtest_pricecontours, it's needed to ensure + # results do not carry-over to the next run, which is not given by using parametrize. + default_conf['protections'] = [ + { + "method": "CooldownPeriod", + "stop_duration": 3, + }] + default_conf['enable_protections'] = True + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + tests = [ + ['sine', 9], + ['raise', 10], + ['lower', 0], + ['sine', 9], + ['raise', 10], + ] + # While buy-signals are unrealistic, running backtesting + # over and over again should not cause different results for [contour, numres] in tests: - simple_backtest(default_conf, contour, numres, mocker, testdatadir) + assert len(simple_backtest(default_conf, contour, mocker, testdatadir)) == numres + + +@pytest.mark.parametrize('protections,contour,expected', [ + (None, 'sine', 35), + (None, 'raise', 19), + (None, 'lower', 0), + (None, 'sine', 35), + (None, 'raise', 19), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'sine', 9), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'raise', 10), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'lower', 0), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'sine', 9), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'raise', 10), +]) +def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir, + protections, contour, expected) -> None: + if protections: + default_conf['protections'] = protections + default_conf['enable_protections'] = True + + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + # While buy-signals are unrealistic, running backtesting + # over and over again should not cause different results + assert len(simple_backtest(default_conf, contour, mocker, testdatadir)) == expected def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): @@ -535,7 +619,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC', datadir=testdatadir) - default_conf['ticker_interval'] = '1m' + default_conf['timeframe'] = '1m' backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = _trend_alternate # Override backtesting.strategy.advise_sell = _trend_alternate # Override @@ -574,7 +658,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) # Remove data for one pair from the beginning of the data data[pair] = data[pair][tres:].reset_index() - default_conf['ticker_interval'] = '5m' + default_conf['timeframe'] = '5m' backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = _trend_alternate_hold # Override @@ -616,7 +700,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats') mocker.patch('freqtrade.optimize.backtesting.show_backtest_results') - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) patched_configuration_load_config_file(mocker, default_conf) @@ -625,7 +709,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): '--config', 'config.json', '--strategy', 'DefaultStrategy', '--datadir', str(testdatadir), - '--ticker-interval', '1m', + '--timeframe', '1m', '--timerange', '1510694220-1510700340', '--enable-position-stacking', '--disable-max-market-positions' @@ -634,16 +718,16 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): start_backtesting(args) # check the logs, that will contain the backtest result exists = [ - 'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', + 'Parameter -i/--timeframe detected ... Using timeframe: 1m ...', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Loading data from 2017-11-14T20:57:00+00:00 ' - 'up to 2017-11-14T22:58:00+00:00 (0 days)..', - 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' - 'up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Loading data from 2017-11-14 20:57:00 ' + 'up to 2017-11-14 22:58:00 (0 days)..', + 'Backtesting with data from 2017-11-14 21:17:00 ' + 'up to 2017-11-14 22:58:00 (0 days)..', 'Parameter --enable-position-stacking detected ...' ] @@ -655,8 +739,8 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): patch_exchange(mocker) - backtestmock = MagicMock() - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + backtestmock = MagicMock(return_value=pd.DataFrame(columns=BT_DATA_COLUMNS + ['profit_abs'])) + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) text_table_mock = MagicMock() @@ -670,6 +754,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): generate_pair_metrics=MagicMock(), generate_sell_reason_stats=sell_reason_mock, generate_strategy_metrics=strat_summary, + generate_daily_stats=MagicMock(), ) patched_configuration_load_config_file(mocker, default_conf) @@ -678,7 +763,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): '--config', 'config.json', '--datadir', str(testdatadir), '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), - '--ticker-interval', '1m', + '--timeframe', '1m', '--timerange', '1510694220-1510700340', '--enable-position-stacking', '--disable-max-market-positions', @@ -697,16 +782,16 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): # check the logs, that will contain the backtest result exists = [ - 'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', + 'Parameter -i/--timeframe detected ... Using timeframe: 1m ...', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Loading data from 2017-11-14T20:57:00+00:00 ' - 'up to 2017-11-14T22:58:00+00:00 (0 days)..', - 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' - 'up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Loading data from 2017-11-14 20:57:00 ' + 'up to 2017-11-14 22:58:00 (0 days)..', + 'Backtesting with data from 2017-11-14 21:17:00 ' + 'up to 2017-11-14 22:58:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy TestStrategyLegacy', @@ -724,13 +809,11 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC'], 'profit_percent': [0.0, 0.0], 'profit_abs': [0.0, 0.0], - 'open_time': pd.to_datetime(['2018-01-29 18:40:00', + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', '2018-01-30 03:30:00', ], utc=True ), - 'close_time': pd.to_datetime(['2018-01-29 20:45:00', + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', '2018-01-30 05:35:00', ], utc=True), - 'open_index': [78, 184], - 'close_index': [125, 192], 'trade_duration': [235, 40], 'open_at_end': [False, False], 'open_rate': [0.104445, 0.10302485], @@ -740,15 +823,13 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat pd.DataFrame({'pair': ['XRP/BTC', 'LTC/BTC', 'ETH/BTC'], 'profit_percent': [0.03, 0.01, 0.1], 'profit_abs': [0.01, 0.02, 0.2], - 'open_time': pd.to_datetime(['2018-01-29 18:40:00', + 'open_date': pd.to_datetime(['2018-01-29 18:40:00', '2018-01-30 03:30:00', '2018-01-30 05:30:00'], utc=True ), - 'close_time': pd.to_datetime(['2018-01-29 20:45:00', + 'close_date': pd.to_datetime(['2018-01-29 20:45:00', '2018-01-30 05:35:00', '2018-01-30 08:30:00'], utc=True), - 'open_index': [78, 184, 185], - 'close_index': [125, 224, 205], 'trade_duration': [47, 40, 20], 'open_at_end': [False, False, False], 'open_rate': [0.104445, 0.10302485, 0.122541], @@ -756,7 +837,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] }), ]) - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) @@ -767,7 +848,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat '--config', 'config.json', '--datadir', str(testdatadir), '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), - '--ticker-interval', '1m', + '--timeframe', '1m', '--timerange', '1510694220-1510700340', '--enable-position-stacking', '--disable-max-market-positions', @@ -780,16 +861,16 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat # check the logs, that will contain the backtest result exists = [ - 'Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', + 'Parameter -i/--timeframe detected ... Using timeframe: 1m ...', 'Ignoring max_open_trades (--disable-max-market-positions was used) ...', 'Parameter --timerange detected: 1510694220-1510700340 ...', f'Using data directory: {testdatadir} ...', 'Using stake_currency: BTC ...', 'Using stake_amount: 0.001 ...', - 'Loading data from 2017-11-14T20:57:00+00:00 ' - 'up to 2017-11-14T22:58:00+00:00 (0 days)..', - 'Backtesting with data from 2017-11-14T21:17:00+00:00 ' - 'up to 2017-11-14T22:58:00+00:00 (0 days)..', + 'Loading data from 2017-11-14 20:57:00 ' + 'up to 2017-11-14 22:58:00 (0 days)..', + 'Backtesting with data from 2017-11-14 21:17:00 ' + 'up to 2017-11-14 22:58:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', 'Running backtesting for Strategy TestStrategyLegacy', diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index a5e468542..188b4aa5f 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -29,7 +29,7 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has('Using data directory: {} ...'.format(config['datadir']), caplog) - assert 'ticker_interval' in config + assert 'timeframe' in config assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog) assert 'timerange' not in config @@ -48,7 +48,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--config', 'config.json', '--strategy', 'DefaultStrategy', '--datadir', '/foo/bar', - '--ticker-interval', '1m', + '--timeframe', '1m', '--timerange', ':100', '--stoplosses=-0.01,-0.10,-0.001' ] @@ -62,8 +62,8 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N assert 'datadir' in config assert config['runmode'] == RunMode.EDGE assert log_has('Using data directory: {} ...'.format(config['datadir']), caplog) - assert 'ticker_interval' in config - assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', + assert 'timeframe' in config + assert log_has('Parameter -i/--timeframe detected ... Using timeframe: 1m ...', caplog) assert 'timerange' in config @@ -105,3 +105,17 @@ def test_edge_init_fee(mocker, edge_conf) -> None: edge_cli = EdgeCli(edge_conf) assert edge_cli.edge.fee == 0.1234 assert fee_mock.call_count == 0 + + +def test_edge_start(mocker, edge_conf) -> None: + mock_calculate = mocker.patch('freqtrade.edge.edge_positioning.Edge.calculate', + return_value=True) + table_mock = mocker.patch('freqtrade.optimize.edge_cli.generate_edge_table') + + patch_exchange(mocker) + edge_conf['stake_amount'] = 20 + + edge_cli = EdgeCli(edge_conf) + edge_cli.start() + assert mock_calculate.call_count == 1 + assert table_mock.call_count == 1 diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 90e047954..82be894d3 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,10 +1,11 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import locale import logging +import re from datetime import datetime from pathlib import Path from typing import Dict, List -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import MagicMock import pandas as pd import pytest @@ -12,48 +13,16 @@ from arrow import Arrow from filelock import Timeout from freqtrade import constants -from freqtrade.commands.optimize_commands import (setup_optimize_configuration, - start_hyperopt) +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt from freqtrade.data.history import load_data from freqtrade.exceptions import DependencyException, OperationalException -from freqtrade.optimize.default_hyperopt import DefaultHyperOpt -from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss from freqtrade.optimize.hyperopt import Hyperopt -from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, - HyperOptResolver) +from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.state import RunMode -from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) - -@pytest.fixture(scope='function') -def hyperopt(default_conf, mocker): - default_conf.update({ - 'spaces': ['default'], - 'hyperopt': 'DefaultHyperOpt', - }) - patch_exchange(mocker) - return Hyperopt(default_conf) - - -@pytest.fixture(scope='function') -def hyperopt_results(): - return pd.DataFrame( - { - 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], - 'profit_percent': [-0.1, 0.2, 0.3], - 'profit_abs': [-0.2, 0.4, 0.6], - 'trade_duration': [10, 30, 10], - 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI], - 'close_time': - [ - datetime(2019, 1, 1, 9, 26, 3, 478039), - datetime(2019, 2, 1, 9, 26, 3, 478039), - datetime(2019, 3, 1, 9, 26, 3, 478039) - ] - } - ) +from .hyperopts.default_hyperopt import DefaultHyperOpt # Functions for recurrent object patching @@ -68,13 +37,14 @@ def create_results(mocker, hyperopt, testdatadir) -> List[Dict]: mocker.patch.object(Path, "is_file", MagicMock(return_value=False)) stat_mock = MagicMock() - stat_mock.st_size = PropertyMock(return_value=1) - mocker.patch.object(Path, "stat", MagicMock(return_value=False)) + stat_mock.st_size = 1 + mocker.patch.object(Path, "stat", MagicMock(return_value=stat_mock)) mocker.patch.object(Path, "unlink", MagicMock(return_value=True)) mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') - return [{'loss': 1, 'result': 'foo', 'params': {}}] + return [{'loss': 1, 'result': 'foo', 'params': {}, 'is_best': True}] def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None: @@ -94,7 +64,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca assert 'pair_whitelist' in config['exchange'] assert 'datadir' in config assert log_has('Using data directory: {} ...'.format(config['datadir']), caplog) - assert 'ticker_interval' in config + assert 'timeframe' in config assert not log_has_re('Parameter -i/--ticker-interval detected .*', caplog) assert 'position_stacking' not in config @@ -117,7 +87,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo '--config', 'config.json', '--hyperopt', 'DefaultHyperOpt', '--datadir', '/foo/bar', - '--ticker-interval', '1m', + '--timeframe', '1m', '--timerange', ':100', '--enable-position-stacking', '--disable-max-market-positions', @@ -136,8 +106,8 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo assert config['runmode'] == RunMode.HYPEROPT assert log_has('Using data directory: {} ...'.format(config['datadir']), caplog) - assert 'ticker_interval' in config - assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...', + assert 'timeframe' in config + assert log_has('Parameter -i/--timeframe detected ... Using timeframe: 1m ...', caplog) assert 'position_stacking' in config @@ -160,7 +130,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo assert log_has('Parameter --print-all detected ...', caplog) -def test_setup_hyperopt_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: +def test_setup_hyperopt_configuration_unlimited_stake_amount(mocker, default_conf) -> None: default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT patched_configuration_load_config_file(mocker, default_conf) @@ -197,10 +167,11 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: "Using populate_sell_trend from the strategy.", caplog) assert log_has("Hyperopt class does not provide populate_buy_trend() method. " "Using populate_buy_trend from the strategy.", caplog) - assert hasattr(x, "ticker_interval") + assert hasattr(x, "ticker_interval") # DEPRECATED + assert hasattr(x, "timeframe") -def test_hyperoptresolver_wrongname(mocker, default_conf, caplog) -> None: +def test_hyperoptresolver_wrongname(default_conf) -> None: default_conf.update({'hyperopt': "NonExistingHyperoptClass"}) with pytest.raises(OperationalException, match=r'Impossible to load Hyperopt.*'): @@ -215,25 +186,7 @@ def test_hyperoptresolver_noname(default_conf): HyperOptResolver.load_hyperopt(default_conf) -def test_hyperoptlossresolver(mocker, default_conf, caplog) -> None: - - hl = DefaultHyperOptLoss - mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver.load_object', - MagicMock(return_value=hl) - ) - x = HyperOptLossResolver.load_hyperoptloss(default_conf) - assert hasattr(x, "hyperopt_loss_function") - - -def test_hyperoptlossresolver_wrongname(mocker, default_conf, caplog) -> None: - default_conf.update({'hyperopt_loss': "NonExistingLossClass"}) - - with pytest.raises(OperationalException, match=r'Impossible to load HyperoptLoss.*'): - HyperOptLossResolver.load_hyperoptloss(default_conf) - - -def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None: +def test_start_not_installed(mocker, default_conf, import_fails) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, default_conf) @@ -244,7 +197,10 @@ def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None 'hyperopt', '--config', 'config.json', '--hyperopt', 'DefaultHyperOpt', - '--epochs', '5' + '--hyperopt-path', + str(Path(__file__).parent / "hyperopts"), + '--epochs', '5', + '--hyperopt-loss', 'SharpeHyperOptLossDaily', ] pargs = get_args(args) @@ -252,9 +208,9 @@ def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None start_hyperopt(pargs) -def test_start(mocker, default_conf, caplog) -> None: +def test_start(mocker, hyperopt_conf, caplog) -> None: start_mock = MagicMock() - patched_configuration_load_config_file(mocker, default_conf) + patched_configuration_load_config_file(mocker, hyperopt_conf) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) patch_exchange(mocker) @@ -262,6 +218,7 @@ def test_start(mocker, default_conf, caplog) -> None: 'hyperopt', '--config', 'config.json', '--hyperopt', 'DefaultHyperOpt', + '--hyperopt-loss', 'SharpeHyperOptLossDaily', '--epochs', '5' ] pargs = get_args(args) @@ -271,8 +228,8 @@ def test_start(mocker, default_conf, caplog) -> None: assert start_mock.call_count == 1 -def test_start_no_data(mocker, default_conf, caplog) -> None: - patched_configuration_load_config_file(mocker, default_conf) +def test_start_no_data(mocker, hyperopt_conf) -> None: + patched_configuration_load_config_file(mocker, hyperopt_conf) mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame)) mocker.patch( 'freqtrade.optimize.hyperopt.get_timerange', @@ -285,6 +242,7 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: 'hyperopt', '--config', 'config.json', '--hyperopt', 'DefaultHyperOpt', + '--hyperopt-loss', 'SharpeHyperOptLossDaily', '--epochs', '5' ] pargs = get_args(args) @@ -292,9 +250,9 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: start_hyperopt(pargs) -def test_start_filelock(mocker, default_conf, caplog) -> None: - start_mock = MagicMock(side_effect=Timeout(Hyperopt.get_lock_filename(default_conf))) - patched_configuration_load_config_file(mocker, default_conf) +def test_start_filelock(mocker, hyperopt_conf, caplog) -> None: + start_mock = MagicMock(side_effect=Timeout(Hyperopt.get_lock_filename(hyperopt_conf))) + patched_configuration_load_config_file(mocker, hyperopt_conf) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) patch_exchange(mocker) @@ -302,6 +260,7 @@ def test_start_filelock(mocker, default_conf, caplog) -> None: 'hyperopt', '--config', 'config.json', '--hyperopt', 'DefaultHyperOpt', + '--hyperopt-loss', 'SharpeHyperOptLossDaily', '--epochs', '5' ] pargs = get_args(args) @@ -309,137 +268,6 @@ def test_start_filelock(mocker, default_conf, caplog) -> None: assert log_has("Another running instance of freqtrade Hyperopt detected.", caplog) -def test_loss_calculation_prefer_correct_trade_count(default_conf, hyperopt_results) -> None: - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, 600, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(hyperopt_results, 600 + 100, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(hyperopt_results, 600 - 100, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over > correct - assert under > correct - - -def test_loss_calculation_prefer_shorter_trades(default_conf, hyperopt_results) -> None: - resultsb = hyperopt_results.copy() - resultsb.loc[1, 'trade_duration'] = 20 - - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - longer = hl.hyperopt_loss_function(hyperopt_results, 100, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - shorter = hl.hyperopt_loss_function(resultsb, 100, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert shorter < longer - - -def test_loss_calculation_has_limited_profit(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, 600, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, 600, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, 600, - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sharpe_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'SharpeHyperOptLoss'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'SortinoHyperOptLoss'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - -def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: - results_over = hyperopt_results.copy() - results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 - results_under = hyperopt_results.copy() - results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - - default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) - hl = HyperOptLossResolver.load_hyperoptloss(default_conf) - correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), - datetime(2019, 1, 1), datetime(2019, 5, 1)) - assert over < correct - assert under > correct - - def test_log_results_if_loss_improves(hyperopt, capsys) -> None: hyperopt.current_best_loss = 2 hyperopt.total_epochs = 2 @@ -481,6 +309,7 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: def test_save_results_saves_epochs(mocker, hyperopt, testdatadir, caplog) -> None: epochs = create_results(mocker, hyperopt, testdatadir) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) + mock_dump_json = mocker.patch('freqtrade.optimize.hyperopt.file_dump_json', return_value=None) results_file = testdatadir / 'optimize' / 'ut_results.pickle' caplog.set_level(logging.DEBUG) @@ -489,6 +318,7 @@ def test_save_results_saves_epochs(mocker, hyperopt, testdatadir, caplog) -> Non hyperopt._save_results() assert log_has(f"1 epoch saved to '{results_file}'.", caplog) mock_dump.assert_called_once() + mock_dump_json.assert_called_once() hyperopt.epochs = epochs + epochs hyperopt._save_results() @@ -505,6 +335,28 @@ def test_read_results_returns_epochs(mocker, hyperopt, testdatadir, caplog) -> N mock_load.assert_called_once() +def test_load_previous_results(mocker, hyperopt, testdatadir, caplog) -> None: + epochs = create_results(mocker, hyperopt, testdatadir) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=epochs) + mocker.patch.object(Path, 'is_file', MagicMock(return_value=True)) + statmock = MagicMock() + statmock.st_size = 5 + # mocker.patch.object(Path, 'stat', MagicMock(return_value=statmock)) + + results_file = testdatadir / 'optimize' / 'ut_results.pickle' + + hyperopt_epochs = hyperopt.load_previous_results(results_file) + + assert hyperopt_epochs == epochs + mock_load.assert_called_once() + + del epochs[0]['is_best'] + mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=epochs) + + with pytest.raises(OperationalException): + hyperopt.load_previous_results(results_file) + + def test_roi_table_generation(hyperopt) -> None: params = { 'roi_t1': 5, @@ -518,8 +370,10 @@ def test_roi_table_generation(hyperopt) -> None: assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} -def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: +def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -544,15 +398,9 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: ) patch_exchange(mocker) # Co-test loading timeframe from strategy - del default_conf['ticker_interval'] - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'default', - 'hyperopt_jobs': 1, }) + del hyperopt_conf['timeframe'] - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) @@ -568,7 +416,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") - assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades'] assert hasattr(hyperopt, "position_stacking") @@ -685,13 +533,36 @@ def test_buy_strategy_generator(hyperopt, testdatadir) -> None: assert 1 in result['buy'] -def test_generate_optimizer(mocker, default_conf) -> None: - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'timerange': None, - 'spaces': 'all', - 'hyperopt_min_trades': 1, - }) +def test_sell_strategy_generator(hyperopt, testdatadir) -> None: + data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) + dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data) + dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], + {'pair': 'UNITTEST/BTC'}) + + populate_sell_trend = hyperopt.custom_hyperopt.sell_strategy_generator( + { + 'sell-adx-value': 20, + 'sell-fastd-value': 75, + 'sell-mfi-value': 80, + 'sell-rsi-value': 20, + 'sell-adx-enabled': True, + 'sell-fastd-enabled': True, + 'sell-mfi-enabled': True, + 'sell-rsi-enabled': True, + 'sell-trigger': 'sell-bb_upper' + } + ) + result = populate_sell_trend(dataframe, {'pair': 'UNITTEST/BTC'}) + # Check if some indicators are generated. We will not test all of them + print(result) + assert 'sell' in result + assert 1 in result['sell'] + + +def test_generate_optimizer(mocker, hyperopt_conf) -> None: + hyperopt_conf.update({'spaces': 'all', + 'hyperopt_min_trades': 1, + }) trades = [ ('TRX/BTC', 0.023117, 0.000233, 100) @@ -743,8 +614,10 @@ def test_generate_optimizer(mocker, default_conf) -> None: } response_expected = { 'loss': 1.9840569076926293, - 'results_explanation': (' 1 trades. Avg profit 2.31%. Total profit 0.00023300 BTC ' - '( 2.31\N{GREEK CAPITAL LETTER SIGMA}%). Avg duration 100.0 min.' + 'results_explanation': (' 1 trades. 1/0/0 Wins/Draws/Losses. ' + 'Avg profit 2.31%. Median profit 2.31%. Total profit ' + '0.00023300 BTC ( 2.31\N{GREEK CAPITAL LETTER SIGMA}%). ' + 'Avg duration 100.0 min.' ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8'), 'params_details': {'buy': {'adx-enabled': False, 'adx-value': 0, @@ -775,56 +648,39 @@ def test_generate_optimizer(mocker, default_conf) -> None: 'trailing_stop_positive_offset': 0.07}}, 'params_dict': optimizer_param, 'results_metrics': {'avg_profit': 2.3117, + 'draws': 0, 'duration': 100.0, + 'losses': 0, + 'winsdrawslosses': ' 1 0 0', + 'median_profit': 2.3117, 'profit': 2.3117, 'total_profit': 0.000233, - 'trade_count': 1}, + 'trade_count': 1, + 'wins': 1}, 'total_profit': 0.00023300 } - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.dimensions = hyperopt.hyperopt_space() generate_optimizer_value = hyperopt.generate_optimizer(list(optimizer_param.values())) assert generate_optimizer_value == response_expected -def test_clean_hyperopt(mocker, default_conf, caplog): +def test_clean_hyperopt(mocker, hyperopt_conf, caplog): patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'default', - 'hyperopt_jobs': 1, - }) + mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True)) unlinkmock = mocker.patch("freqtrade.optimize.hyperopt.Path.unlink", MagicMock()) - h = Hyperopt(default_conf) + h = Hyperopt(hyperopt_conf) assert unlinkmock.call_count == 2 assert log_has(f"Removing `{h.data_pickle_file}`.", caplog) -def test_continue_hyperopt(mocker, default_conf, caplog): - patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'default', - 'hyperopt_jobs': 1, - 'hyperopt_continue': True - }) - mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True)) - unlinkmock = mocker.patch("freqtrade.optimize.hyperopt.Path.unlink", MagicMock()) - Hyperopt(default_conf) - - assert unlinkmock.call_count == 0 - assert log_has("Continuing on previous hyperopt results.", caplog) - - -def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: +def test_print_json_spaces_all(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') + mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -854,16 +710,12 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: ) patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'all', - 'hyperopt_jobs': 1, - 'print_json': True, - }) + hyperopt_conf.update({'spaces': 'all', + 'hyperopt_jobs': 1, + 'print_json': True, + }) - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) @@ -882,8 +734,9 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: assert dumper.call_count == 2 -def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None: +def test_print_json_spaces_default(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -912,16 +765,9 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None ) patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'default', - 'hyperopt_jobs': 1, - 'print_json': True, - }) + hyperopt_conf.update({'print_json': True}) - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) @@ -936,8 +782,9 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None assert dumper.call_count == 2 -def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: +def test_print_json_spaces_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -962,16 +809,12 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> ) patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'roi stoploss', - 'hyperopt_jobs': 1, - 'print_json': True, - }) + hyperopt_conf.update({'spaces': 'roi stoploss', + 'hyperopt_jobs': 1, + 'print_json': True, + }) - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) @@ -986,8 +829,9 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> assert dumper.call_count == 2 -def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) -> None: +def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -1011,14 +855,9 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) ) patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'roi stoploss', - 'hyperopt_jobs': 1, }) + hyperopt_conf.update({'spaces': 'roi stoploss'}) - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) @@ -1039,12 +878,13 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") - assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades'] assert hasattr(hyperopt, "position_stacking") -def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) -> None: +def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -1054,14 +894,9 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) - patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'all', - 'hyperopt_jobs': 1, }) + hyperopt_conf.update({'spaces': 'all', }) - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) @@ -1074,8 +909,9 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) - hyperopt.start() -def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: +def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -1099,14 +935,9 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: ) patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'buy', - 'hyperopt_jobs': 1, }) + hyperopt_conf.update({'spaces': 'buy'}) - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) @@ -1127,12 +958,13 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") - assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades'] assert hasattr(hyperopt, "position_stacking") -def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None: +def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -1156,14 +988,9 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None ) patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': 'sell', - 'hyperopt_jobs': 1, }) + hyperopt_conf.update({'spaces': 'sell', }) - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) @@ -1184,7 +1011,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") assert hasattr(hyperopt, "max_open_trades") - assert hyperopt.max_open_trades == default_conf['max_open_trades'] + assert hyperopt.max_open_trades == hyperopt_conf['max_open_trades'] assert hasattr(hyperopt, "position_stacking") @@ -1194,8 +1021,9 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None ('sell_strategy_generator', 'sell'), ('sell_indicator_space', 'sell'), ]) -def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, method, space) -> None: +def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) + mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', MagicMock(return_value=(MagicMock(), None))) mocker.patch( @@ -1205,14 +1033,9 @@ def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, metho patch_exchange(mocker) - default_conf.update({'config': 'config.json.example', - 'hyperopt': 'DefaultHyperOpt', - 'epochs': 1, - 'timerange': None, - 'spaces': space, - 'hyperopt_jobs': 1, }) + hyperopt_conf.update({'spaces': space}) - hyperopt = Hyperopt(default_conf) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) @@ -1220,3 +1043,39 @@ def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, metho with pytest.raises(OperationalException, match=f"The '{space}' space is included into *"): hyperopt.start() + + +def test_print_epoch_details(capsys): + test_result = { + 'params_details': { + 'trailing': { + 'trailing_stop': True, + 'trailing_stop_positive': 0.02, + 'trailing_stop_positive_offset': 0.04, + 'trailing_only_offset_is_reached': True + }, + 'roi': { + 0: 0.18, + 90: 0.14, + 225: 0.05, + 430: 0}, + }, + 'results_explanation': 'foo result', + 'is_initial_point': False, + 'total_profit': 0, + 'current_epoch': 2, # This starts from 1 (in a human-friendly manner) + 'is_best': True + } + + Hyperopt.print_epoch_details(test_result, 5, False, no_header=True) + captured = capsys.readouterr() + assert '# Trailing stop:' in captured.out + # re.match(r"Pairs for .*", captured.out) + assert re.search(r'^\s+trailing_stop = True$', captured.out, re.MULTILINE) + assert re.search(r'^\s+trailing_stop_positive = 0.02$', captured.out, re.MULTILINE) + assert re.search(r'^\s+trailing_stop_positive_offset = 0.04$', captured.out, re.MULTILINE) + assert re.search(r'^\s+trailing_only_offset_is_reached = True$', captured.out, re.MULTILINE) + + assert '# ROI table:' in captured.out + assert re.search(r'^\s+minimal_roi = \{$', captured.out, re.MULTILINE) + assert re.search(r'^\s+\"90\"\:\s0.14,\s*$', captured.out, re.MULTILINE) diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py new file mode 100644 index 000000000..63012ee48 --- /dev/null +++ b/tests/optimize/test_hyperoptloss.py @@ -0,0 +1,165 @@ +from datetime import datetime +from unittest.mock import MagicMock + +import pytest + +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.default_hyperopt_loss import ShortTradeDurHyperOptLoss +from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver + + +def test_hyperoptlossresolver_noname(default_conf): + with pytest.raises(OperationalException, + match="No Hyperopt loss set. Please use `--hyperopt-loss` to specify " + "the Hyperopt-Loss class to use."): + HyperOptLossResolver.load_hyperoptloss(default_conf) + + +def test_hyperoptlossresolver(mocker, default_conf) -> None: + + hl = ShortTradeDurHyperOptLoss + mocker.patch( + 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver.load_object', + MagicMock(return_value=hl) + ) + default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) + x = HyperOptLossResolver.load_hyperoptloss(default_conf) + assert hasattr(x, "hyperopt_loss_function") + + +def test_hyperoptlossresolver_wrongname(default_conf) -> None: + default_conf.update({'hyperopt_loss': "NonExistingLossClass"}) + + with pytest.raises(OperationalException, match=r'Impossible to load HyperoptLoss.*'): + HyperOptLossResolver.load_hyperoptloss(default_conf) + + +def test_loss_calculation_prefer_correct_trade_count(hyperopt_conf, hyperopt_results) -> None: + hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(hyperopt_results, 600 + 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(hyperopt_results, 600 - 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over > correct + assert under > correct + + +def test_loss_calculation_prefer_shorter_trades(hyperopt_conf, hyperopt_results) -> None: + resultsb = hyperopt_results.copy() + resultsb.loc[1, 'trade_duration'] = 20 + + hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) + longer = hl.hyperopt_loss_function(hyperopt_results, 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + shorter = hl.hyperopt_loss_function(resultsb, 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert shorter < longer + + +def test_loss_calculation_has_limited_profit(hyperopt_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + hl = HyperOptLossResolver.load_hyperoptloss(hyperopt_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sharpe_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SharpeHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index 175405e4c..a0e1932ff 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -1,19 +1,28 @@ +import re +from datetime import timedelta from pathlib import Path import pandas as pd import pytest from arrow import Arrow +from freqtrade.configuration import TimeRange +from freqtrade.constants import LAST_BT_RESULT_FN +from freqtrade.data import history +from freqtrade.data.btanalysis import get_latest_backtest_filename, load_backtest_data from freqtrade.edge import PairInfo -from freqtrade.optimize.optimize_reports import ( - generate_pair_metrics, generate_edge_table, generate_sell_reason_stats, - text_table_bt_results, text_table_sell_reason, generate_strategy_metrics, - text_table_strategy, store_backtest_result) +from freqtrade.optimize.optimize_reports import (generate_backtest_stats, generate_daily_stats, + generate_edge_table, generate_pair_metrics, + generate_sell_reason_stats, + generate_strategy_metrics, store_backtest_stats, + text_table_bt_results, text_table_sell_reason, + text_table_strategy) +from freqtrade.resolvers.strategy_resolver import StrategyResolver from freqtrade.strategy.interface import SellType -from tests.conftest import patch_exchange +from tests.data.test_history import _backup_file, _clean_test_file -def test_text_table_bt_results(default_conf, mocker): +def test_text_table_bt_results(): results = pd.DataFrame( { @@ -43,7 +52,126 @@ def test_text_table_bt_results(default_conf, mocker): assert text_table_bt_results(pair_results, stake_currency='BTC') == result_str -def test_generate_pair_metrics(default_conf, mocker): +def test_generate_backtest_stats(default_conf, testdatadir): + default_conf.update({'strategy': 'DefaultStrategy'}) + StrategyResolver.load_strategy(default_conf) + + results = {'DefStrat': { + 'results': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", + "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], + "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True], + "sell_reason": [SellType.ROI, SellType.STOP_LOSS, + SellType.ROI, SellType.FORCE_SELL] + }), + 'config': default_conf, + 'locks': []} + } + timerange = TimeRange.parse_timerange('1510688220-1510700340') + min_date = Arrow.fromtimestamp(1510688220) + max_date = Arrow.fromtimestamp(1510700340) + btdata = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, + fill_up_missing=True) + + stats = generate_backtest_stats(btdata, results, min_date, max_date) + assert isinstance(stats, dict) + assert 'strategy' in stats + assert 'DefStrat' in stats['strategy'] + assert 'strategy_comparison' in stats + strat_stats = stats['strategy']['DefStrat'] + assert strat_stats['backtest_start'] == min_date.datetime + assert strat_stats['backtest_end'] == max_date.datetime + assert strat_stats['total_trades'] == len(results['DefStrat']['results']) + # Above sample had no loosing trade + assert strat_stats['max_drawdown'] == 0.0 + + results = {'DefStrat': { + 'results': pd.DataFrame( + {"pair": ["UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_percent": [0.003312, 0.010801, -0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, -0.000014, 0.000003], + "open_date": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_date": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.0032903, 0.003217], + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True], + "sell_reason": [SellType.ROI, SellType.STOP_LOSS, + SellType.ROI, SellType.FORCE_SELL] + }), + 'config': default_conf} + } + + assert strat_stats['max_drawdown'] == 0.0 + assert strat_stats['drawdown_start'] == Arrow.fromtimestamp(0).datetime + assert strat_stats['drawdown_end'] == Arrow.fromtimestamp(0).datetime + assert strat_stats['drawdown_end_ts'] == 0 + assert strat_stats['drawdown_start_ts'] == 0 + assert strat_stats['pairlist'] == ['UNITTEST/BTC'] + + # Test storing stats + filename = Path(testdatadir / 'btresult.json') + filename_last = Path(testdatadir / LAST_BT_RESULT_FN) + _backup_file(filename_last, copy_file=True) + assert not filename.is_file() + + store_backtest_stats(filename, stats) + + # get real Filename (it's btresult-.json) + last_fn = get_latest_backtest_filename(filename_last.parent) + assert re.match(r"btresult-.*\.json", last_fn) + + filename1 = (testdatadir / last_fn) + assert filename1.is_file() + content = filename1.read_text() + assert 'max_drawdown' in content + assert 'strategy' in content + assert 'pairlist' in content + + assert filename_last.is_file() + + _clean_test_file(filename_last) + filename1.unlink() + + +def test_store_backtest_stats(testdatadir, mocker): + + dump_mock = mocker.patch('freqtrade.optimize.optimize_reports.file_dump_json') + + store_backtest_stats(testdatadir, {}) + + assert dump_mock.call_count == 2 + assert isinstance(dump_mock.call_args_list[0][0][0], Path) + assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir/'backtest-result')) + + dump_mock.reset_mock() + filename = testdatadir / 'testresult.json' + store_backtest_stats(filename, {}) + assert dump_mock.call_count == 2 + assert isinstance(dump_mock.call_args_list[0][0][0], Path) + # result will be testdatadir / testresult-.json + assert str(dump_mock.call_args_list[0][0][0]).startswith(str(testdatadir / 'testresult')) + + +def test_generate_pair_metrics(): results = pd.DataFrame( { @@ -68,7 +196,30 @@ def test_generate_pair_metrics(default_conf, mocker): pytest.approx(pair_results[-1]['profit_sum_pct']) == pair_results[-1]['profit_sum'] * 100) -def test_text_table_sell_reason(default_conf): +def test_generate_daily_stats(testdatadir): + + filename = testdatadir / "backtest-result_new.json" + bt_data = load_backtest_data(filename) + res = generate_daily_stats(bt_data) + assert isinstance(res, dict) + assert round(res['backtest_best_day'], 4) == 0.1796 + assert round(res['backtest_worst_day'], 4) == -0.1468 + assert res['winning_days'] == 14 + assert res['draw_days'] == 4 + assert res['losing_days'] == 3 + assert res['winner_holding_avg'] == timedelta(seconds=1440) + assert res['loser_holding_avg'] == timedelta(days=1, seconds=21420) + + # Select empty dataframe! + res = generate_daily_stats(bt_data.loc[bt_data['open_date'] == '2000-01-01', :]) + assert isinstance(res, dict) + assert round(res['backtest_best_day'], 4) == 0.0 + assert res['winning_days'] == 0 + assert res['draw_days'] == 0 + assert res['losing_days'] == 0 + + +def test_text_table_sell_reason(): results = pd.DataFrame( { @@ -100,7 +251,7 @@ def test_text_table_sell_reason(default_conf): stake_currency='BTC') == result_str -def test_generate_sell_reason_stats(default_conf): +def test_generate_sell_reason_stats(): results = pd.DataFrame( { @@ -135,9 +286,10 @@ def test_generate_sell_reason_stats(default_conf): assert stop_result['profit_mean_pct'] == round(stop_result['profit_mean'] * 100, 2) -def test_text_table_strategy(default_conf, mocker): +def test_text_table_strategy(default_conf): + default_conf['max_open_trades'] = 2 results = {} - results['TestStrategy1'] = pd.DataFrame( + results['TestStrategy1'] = {'results': pd.DataFrame( { 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], 'profit_percent': [0.1, 0.2, 0.3], @@ -148,8 +300,8 @@ def test_text_table_strategy(default_conf, mocker): 'losses': [0, 0, 1], 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] } - ) - results['TestStrategy2'] = pd.DataFrame( + ), 'config': default_conf} + results['TestStrategy2'] = {'results': pd.DataFrame( { 'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'], 'profit_percent': [0.4, 0.2, 0.3], @@ -160,7 +312,7 @@ def test_text_table_strategy(default_conf, mocker): 'losses': [0, 0, 1], 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] } - ) + ), 'config': default_conf} result_str = ( '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' @@ -173,14 +325,12 @@ def test_text_table_strategy(default_conf, mocker): ' 45.00 | 0:20:00 | 3 | 0 | 0 |' ) - strategy_results = generate_strategy_metrics(stake_currency='BTC', - max_open_trades=2, - all_results=results) + strategy_results = generate_strategy_metrics(all_results=results) assert text_table_strategy(strategy_results, 'BTC') == result_str -def test_generate_edge_table(edge_conf, mocker): +def test_generate_edge_table(): results = {} results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) @@ -188,77 +338,3 @@ def test_generate_edge_table(edge_conf, mocker): assert generate_edge_table(results).count('| ETH/BTC |') == 1 assert generate_edge_table(results).count( '| Risk Reward Ratio | Required Risk Reward | Expectancy |') == 1 - - -def test_backtest_record(default_conf, fee, mocker): - names = [] - records = [] - patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - mocker.patch( - 'freqtrade.optimize.optimize_reports.file_dump_json', - new=lambda n, r: (names.append(n), records.append(r)) - ) - - results = {'DefStrat': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", - "UNITTEST/BTC", "UNITTEST/BTC"], - "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780], - "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], - "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], - "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], - "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], - "open_index": [1, 119, 153, 185], - "close_index": [118, 151, 184, 199], - "trade_duration": [123, 34, 31, 14], - "open_at_end": [False, False, False, True], - "sell_reason": [SellType.ROI, SellType.STOP_LOSS, - SellType.ROI, SellType.FORCE_SELL] - })} - store_backtest_result(Path("backtest-result.json"), results) - # Assert file_dump_json was only called once - assert names == [Path('backtest-result.json')] - records = records[0] - # Ensure records are of correct type - assert len(records) == 4 - - # reset test to test with strategy name - names = [] - records = [] - results['Strat'] = results['DefStrat'] - results['Strat2'] = results['DefStrat'] - store_backtest_result(Path("backtest-result.json"), results) - assert names == [ - Path('backtest-result-DefStrat.json'), - Path('backtest-result-Strat.json'), - Path('backtest-result-Strat2.json'), - ] - records = records[0] - # Ensure records are of correct type - assert len(records) == 4 - - # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) - # Below follows just a typecheck of the schema/type of trade-records - oix = None - for (pair, profit, date_buy, date_sell, buy_index, dur, - openr, closer, open_at_end, sell_reason) in records: - assert pair == 'UNITTEST/BTC' - assert isinstance(profit, float) - # FIX: buy/sell should be converted to ints - assert isinstance(date_buy, float) - assert isinstance(date_sell, float) - assert isinstance(openr, float) - assert isinstance(closer, float) - assert isinstance(open_at_end, bool) - assert isinstance(sell_reason, str) - isinstance(buy_index, pd._libs.tslib.Timestamp) - if oix: - assert buy_index > oix - oix = buy_index - assert dur > 0 diff --git a/tests/plugins/__init__.py b/tests/plugins/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/pairlist/test_pairlist.py b/tests/plugins/test_pairlist.py similarity index 51% rename from tests/pairlist/test_pairlist.py rename to tests/plugins/test_pairlist.py index 421f06911..1795fc27f 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -6,7 +6,7 @@ import pytest from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.pairlistmanager import PairListManager +from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver from tests.conftest import get_patched_freqtradebot, log_has, log_has_re @@ -57,6 +57,31 @@ def whitelist_conf_2(default_conf): return default_conf +@pytest.fixture(scope="function") +def whitelist_conf_agefilter(default_conf): + default_conf['stake_currency'] = 'BTC' + default_conf['exchange']['pair_whitelist'] = [ + 'ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC', + 'BTT/BTC', 'HOT/BTC', 'FUEL/BTC', 'XRP/BTC' + ] + default_conf['exchange']['pair_blacklist'] = [ + 'BLK/BTC' + ] + default_conf['pairlists'] = [ + { + "method": "VolumePairList", + "number_assets": 5, + "sort_key": "quoteVolume", + "refresh_period": 0, + }, + { + "method": "AgeFilter", + "min_days_listed": 2 + } + ] + return default_conf + + @pytest.fixture(scope="function") def static_pl_conf(whitelist_conf): whitelist_conf['pairlists'] = [ @@ -67,7 +92,7 @@ def static_pl_conf(whitelist_conf): return whitelist_conf -def test_log_on_refresh(mocker, static_pl_conf, markets, tickers): +def test_log_cached(mocker, static_pl_conf, markets, tickers): mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), exchange_has=MagicMock(return_value=True), @@ -77,14 +102,14 @@ def test_log_on_refresh(mocker, static_pl_conf, markets, tickers): logmock = MagicMock() # Assign starting whitelist pl = freqtrade.pairlists._pairlist_handlers[0] - pl.log_on_refresh(logmock, 'Hello world') + pl.log_once('Hello world', logmock) assert logmock.call_count == 1 - pl.log_on_refresh(logmock, 'Hello world') + pl.log_once('Hello world', logmock) assert logmock.call_count == 1 assert pl._log_cache.currsize == 1 assert ('Hello world',) in pl._log_cache._Cache__data - pl.log_on_refresh(logmock, 'Hello world2') + pl.log_once('Hello world2', logmock) assert logmock.call_count == 2 assert pl._log_cache.currsize == 2 @@ -165,7 +190,7 @@ def test_refresh_pairlist_dynamic_2(mocker, shitcoinmarkets, tickers, whitelist_ ) # Remove caching of ticker data to emulate changing volume by the time of second call mocker.patch.multiple( - 'freqtrade.pairlist.pairlistmanager.PairListManager', + 'freqtrade.plugins.pairlistmanager.PairListManager', _get_cached_tickers=MagicMock(return_value=tickers_dict), ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_2) @@ -206,11 +231,8 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # VolumePairList only ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), - # Different sorting depending on quote or bid volume - ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']), + "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']), # No pair for ETH, VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "ETH", []), @@ -220,19 +242,24 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # No pair for ETH, all handlers ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 2}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.03}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, - {"method": "ShuffleFilter"}], + {"method": "ShuffleFilter"}, {"method": "PerformanceFilter"}], "ETH", []), + # AgeFilter and VolumePairList (require 2 days only, all should pass age test) + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 2}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), + # AgeFilter and VolumePairList (require 10 days, all should fail age test) + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 10}], + "BTC", []), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), - # Precisionfilter bid - ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, - {"method": "PrecisionFilter"}], - "BTC", ['FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), # PriceFilter and VolumePairList ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], @@ -241,11 +268,16 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], "USDT", ['ETH/USDT', 'NANO/USDT']), - # Hot is removed by precision_filter, Fuel by low_price_filter. + # Hot is removed by precision_filter, Fuel by low_price_ratio, Ripple by min_price. ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, - {"method": "PriceFilter", "low_price_ratio": 0.02}], - "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + {"method": "PriceFilter", "low_price_ratio": 0.02, "min_price": 0.01}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + # Hot is removed by precision_filter, Fuel by low_price_ratio, Ethereum by max_price. + ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"}, + {"method": "PrecisionFilter"}, + {"method": "PriceFilter", "low_price_ratio": 0.02, "max_price": 0.05}], + "BTC", ['TKN/BTC', 'LTC/BTC', 'XRP/BTC']), # HOT and XRP are removed because below 1250 quoteVolume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", "min_value": 1250}], @@ -254,9 +286,6 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "StaticPairList"}], "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), # Static Pairlist before VolumePairList - sorting changes - ([{"method": "StaticPairList"}, - {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], - "BTC", ['HOT/BTC', 'TKN/BTC', 'ETH/BTC']), # SpreadFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], @@ -264,15 +293,18 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # ShuffleFilter ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter", "seed": 77}], - "USDT", ['ETH/USDT', 'ADAHALF/USDT', 'NANO/USDT']), + "USDT", ['ADADOUBLE/USDT', 'ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']), # ShuffleFilter, other seed ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter", "seed": 42}], - "USDT", ['NANO/USDT', 'ETH/USDT', 'ADAHALF/USDT']), + "USDT", ['ADAHALF/USDT', 'NANO/USDT', 'ADADOUBLE/USDT', 'ETH/USDT']), # ShuffleFilter, no seed ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "ShuffleFilter"}], - "USDT", 3), # whitelist_result is integer -- check only lenght of randomized pairlist + "USDT", 3), # whitelist_result is integer -- check only length of randomized pairlist + # AgeFilter only + ([{"method": "AgeFilter", "min_days_listed": 2}], + "BTC", 'filter_at_the_beginning'), # OperationalException expected # PrecisionFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "PrecisionFilter"}], @@ -282,7 +314,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "BTC", 'filter_at_the_beginning'), # OperationalException expected # PriceFilter after StaticPairList ([{"method": "StaticPairList"}, - {"method": "PriceFilter", "low_price_ratio": 0.02}], + {"method": "PriceFilter", "low_price_ratio": 0.02, "min_price": 0.000001, "max_price": 0.1}], "BTC", ['ETH/BTC', 'TKN/BTC']), # PriceFilter only ([{"method": "PriceFilter", "low_price_ratio": 0.02}], @@ -294,6 +326,13 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # ShuffleFilter only ([{"method": "ShuffleFilter", "seed": 42}], "BTC", 'filter_at_the_beginning'), # OperationalException expected + # PerformanceFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "PerformanceFilter"}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), + # PerformanceFilter only + ([{"method": "PerformanceFilter"}], + "BTC", 'filter_at_the_beginning'), # OperationalException expected # SpreadFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], @@ -302,16 +341,31 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "SpreadFilter", "max_spread_ratio": 0.005}], "BTC", 'filter_at_the_beginning'), # OperationalException expected # Static Pairlist after VolumePairList, on a non-first position - ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "StaticPairList"}], - "BTC", 'static_in_the_middle'), + "BTC", 'static_in_the_middle'), + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "PriceFilter", "low_price_ratio": 0.02}], + "USDT", ['ETH/USDT', 'NANO/USDT']), + ([{"method": "StaticPairList"}, + {"method": "RangeStabilityFilter", "lookback_days": 10, + "min_rate_of_change": 0.01, "refresh_period": 1440}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, - pairlists, base_currency, whitelist_result, - caplog) -> None: + ohlcv_history, pairlists, base_currency, + whitelist_result, caplog) -> None: whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history, + ('HOT/BTC', '1d'): ohlcv_history, + } + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) if whitelist_result == 'static_in_the_middle': @@ -324,7 +378,16 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch.multiple('freqtrade.exchange.Exchange', get_tickers=tickers, - markets=PropertyMock(return_value=shitcoinmarkets), + markets=PropertyMock(return_value=shitcoinmarkets) + ) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) + + # Provide for PerformanceFilter's dependency + mocker.patch.multiple('freqtrade.persistence.Trade', + get_overall_performance=MagicMock(return_value=[]) ) # Set whitelist_result to None if pairlist is invalid and should produce exception @@ -346,11 +409,19 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t len(whitelist) == whitelist_result for pairlist in pairlists: + if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \ + len(ohlcv_history) <= pairlist['min_days_listed']: + assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' + r'.* day.*', caplog) if pairlist['method'] == 'PrecisionFilter' and whitelist_result: assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' r'would be <= stop limit.*', caplog) if pairlist['method'] == 'PriceFilter' and whitelist_result: assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or + log_has_re(r'^Removed .* from whitelist, ' + r'because last price < .*%$', caplog) or + log_has_re(r'^Removed .* from whitelist, ' + r'because last price > .*%$', caplog) or log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] " r"is empty.*", caplog)) if pairlist['method'] == 'VolumePairList': @@ -362,6 +433,17 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t assert not log_has(logmsg, caplog) +def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: + whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}] + del whitelist_conf['stoploss'] + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + with pytest.raises(OperationalException, + match=r"PrecisionFilter can only work with stoploss defined\..*"): + PairListManager(MagicMock, whitelist_conf) + + def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}] @@ -400,7 +482,9 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): # BCH/BTC not available (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), # BTT/BTC is inactive - (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") + (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active"), + # XLTCUSDT is not a valid pair + (['ETH/BTC', 'TKN/BTC', 'XLTCUSDT'], "is not tradable with Freqtrade"), ]) def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist, whitelist, caplog, log_message, tickers): @@ -421,7 +505,24 @@ def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist assert log_message in caplog.text -def test_volumepairlist_invalid_sortvalue(mocker, markets, whitelist_conf): +@pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) +def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, pairlist, tickers): + whitelist_conf['pairlists'][0]['method'] = pairlist + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True) + + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=None), + get_tickers=tickers + ) + # Assign starting whitelist + pairlist_handler = freqtrade.pairlists._pairlist_handlers[0] + with pytest.raises(OperationalException, match=r'Markets not loaded.*'): + pairlist_handler._whitelist_for_active_markets(['ETH/BTC']) + + +def test_volumepairlist_invalid_sortvalue(mocker, whitelist_conf): whitelist_conf['pairlists'][0].update({"sort_key": "asdf"}) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -451,7 +552,191 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): assert freqtrade.pairlists._pairlist_handlers[0]._last_refresh == lrf -def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): +def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tickers): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'AgeFilter', 'min_days_listed': -1}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + with pytest.raises(OperationalException, + match=r'AgeFilter requires min_days_listed to be >= 1'): + get_patched_freqtradebot(mocker, default_conf) + + +def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'AgeFilter', 'min_days_listed': 99999}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + with pytest.raises(OperationalException, + match=r'AgeFilter requires min_days_listed to not exceed ' + r'exchange max request size \([0-9]+\)'): + get_patched_freqtradebot(mocker, default_conf) + + +def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + } + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) + + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 + # freqtrade.config['exchange']['pair_whitelist'].append('HOT/BTC') + + previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == 3 + # Called once for XRP/BTC + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 + + +def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'RangeStabilityFilter', 'lookback_days': 99999}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + with pytest.raises(OperationalException, + match=r'RangeStabilityFilter requires lookback_days to not exceed ' + r'exchange max request size \([0-9]+\)'): + get_patched_freqtradebot(mocker, default_conf) + + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'RangeStabilityFilter', 'lookback_days': 0}] + + with pytest.raises(OperationalException, + match='RangeStabilityFilter requires lookback_days to be >= 1'): + get_patched_freqtradebot(mocker, default_conf) + + +@pytest.mark.parametrize('min_rate_of_change,expected_length', [ + (0.01, 5), + (0.05, 0), # Setting rate_of_change to 5% removes all pairs from the whitelist. +]) +def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history, + min_rate_of_change, expected_length): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'RangeStabilityFilter', 'lookback_days': 2, + 'min_rate_of_change': min_rate_of_change}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history, + ('HOT/BTC', '1d'): ohlcv_history, + ('BLK/BTC', '1d'): ohlcv_history, + } + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == expected_length + assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 + + previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count + freqtrade.pairlists.refresh_pairlist() + assert len(freqtrade.pairlists.whitelist) == expected_length + # Should not have increased since first call. + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + + +@pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [ + ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010, + "max_price": 1.0}, + "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below " + "0.1% or below 0.00000010 or above 1.00000000.'}]", + None + ), + ({"method": "PriceFilter", "low_price_ratio": 0.001, "min_price": 0.00000010}, + "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.1% or below 0.00000010.'}]", + None + ), + ({"method": "PriceFilter", "low_price_ratio": 0.001, "max_price": 1.00010000}, + "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.1% or above 1.00010000.'}]", + None + ), + ({"method": "PriceFilter", "min_price": 0.00002000}, + "[{'PriceFilter': 'PriceFilter - Filtering pairs priced below 0.00002000.'}]", + None + ), + ({"method": "PriceFilter"}, + "[{'PriceFilter': 'PriceFilter - No price filters configured.'}]", + None + ), + ({"method": "PriceFilter", "low_price_ratio": -0.001}, + None, + "PriceFilter requires low_price_ratio to be >= 0" + ), # OperationalException expected + ({"method": "PriceFilter", "min_price": -0.00000010}, + None, + "PriceFilter requires min_price to be >= 0" + ), # OperationalException expected + ({"method": "PriceFilter", "max_price": -1.00010000}, + None, + "PriceFilter requires max_price to be >= 0" + ), # OperationalException expected + ({"method": "RangeStabilityFilter", "lookback_days": 10, "min_rate_of_change": 0.01}, + "[{'RangeStabilityFilter': 'RangeStabilityFilter - Filtering pairs with rate of change below " + "0.01 over the last days.'}]", + None + ), +]) +def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, + desc_expected, exception_expected): + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True) + ) + whitelist_conf['pairlists'] = [pairlistconfig] + + if desc_expected is not None: + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + short_desc = str(freqtrade.pairlists.short_desc()) + assert short_desc == desc_expected + else: # OperationalException expected + with pytest.raises(OperationalException, + match=exception_expected): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + + +def test_pairlistmanager_no_pairlist(mocker, whitelist_conf): mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) whitelist_conf['pairlists'] = [] @@ -459,3 +744,63 @@ def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): with pytest.raises(OperationalException, match=r"No Pairlist Handlers defined"): get_patched_freqtradebot(mocker, whitelist_conf) + + +@pytest.mark.parametrize("pairlists,pair_allowlist,overall_performance,allowlist_result", [ + # No trades yet + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], [], ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + # Happy path: Descending order, all values filled + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC'], + [{'pair': 'TKN/BTC', 'profit': 5, 'count': 3}, {'pair': 'ETH/BTC', 'profit': 4, 'count': 2}], + ['TKN/BTC', 'ETH/BTC']), + # Performance data outside allow list ignored + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC'], + [{'pair': 'OTHER/BTC', 'profit': 5, 'count': 3}, + {'pair': 'ETH/BTC', 'profit': 4, 'count': 2}], + ['ETH/BTC', 'TKN/BTC']), + # Partial performance data missing and sorted between positive and negative profit + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], + [{'pair': 'ETH/BTC', 'profit': -5, 'count': 100}, + {'pair': 'TKN/BTC', 'profit': 4, 'count': 2}], + ['TKN/BTC', 'LTC/BTC', 'ETH/BTC']), + # Tie in performance data broken by count (ascending) + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], + [{'pair': 'LTC/BTC', 'profit': -5.01, 'count': 101}, + {'pair': 'TKN/BTC', 'profit': -5.01, 'count': 2}, + {'pair': 'ETH/BTC', 'profit': -5.01, 'count': 100}], + ['TKN/BTC', 'ETH/BTC', 'LTC/BTC']), + # Tie in performance and count, broken by alphabetical sort + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], + [{'pair': 'LTC/BTC', 'profit': -5.01, 'count': 1}, + {'pair': 'TKN/BTC', 'profit': -5.01, 'count': 1}, + {'pair': 'ETH/BTC', 'profit': -5.01, 'count': 1}], + ['ETH/BTC', 'LTC/BTC', 'TKN/BTC']), +]) +def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, overall_performance, + allowlist_result, tickers, markets, ohlcv_history_list): + allowlist_conf = whitelist_conf + allowlist_conf['pairlists'] = pairlists + allowlist_conf['exchange']['pair_whitelist'] = pair_allowlist + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + freqtrade = get_patched_freqtradebot(mocker, allowlist_conf) + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_tickers=tickers, + markets=PropertyMock(return_value=markets) + ) + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), + ) + mocker.patch.multiple('freqtrade.persistence.Trade', + get_overall_performance=MagicMock(return_value=overall_performance), + ) + freqtrade.pairlists.refresh_pairlist() + allowlist = freqtrade.pairlists.whitelist + assert allowlist == allowlist_result diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py new file mode 100644 index 000000000..bd103b21e --- /dev/null +++ b/tests/plugins/test_pairlocks.py @@ -0,0 +1,116 @@ +from datetime import datetime, timedelta, timezone + +import arrow +import pytest + +from freqtrade.persistence import PairLocks +from freqtrade.persistence.models import PairLock + + +@pytest.mark.parametrize('use_db', (False, True)) +@pytest.mark.usefixtures("init_persistence") +def test_PairLocks(use_db): + PairLocks.timeframe = '5m' + # No lock should be present + if use_db: + assert len(PairLock.query.all()) == 0 + else: + PairLocks.use_db = False + + assert PairLocks.use_db == use_db + + pair = 'ETH/BTC' + assert not PairLocks.is_pair_locked(pair) + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + # ETH/BTC locked for 4 minutes + assert PairLocks.is_pair_locked(pair) + + # XRP/BTC should not be locked now + pair = 'XRP/BTC' + assert not PairLocks.is_pair_locked(pair) + # Unlocking a pair that's not locked should not raise an error + PairLocks.unlock_pair(pair) + + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + assert PairLocks.is_pair_locked(pair) + + # Get both locks from above + locks = PairLocks.get_pair_locks(None) + assert len(locks) == 2 + + # Unlock original pair + pair = 'ETH/BTC' + PairLocks.unlock_pair(pair) + assert not PairLocks.is_pair_locked(pair) + assert not PairLocks.is_global_lock() + + pair = 'BTC/USDT' + # Lock until 14:30 + lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc) + PairLocks.lock_pair(pair, lock_time) + + assert not PairLocks.is_pair_locked(pair) + assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-10)) + assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-10)) + assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) + assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50)) + + # Should not be locked after time expired + assert not PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=10)) + + locks = PairLocks.get_pair_locks(pair, lock_time + timedelta(minutes=-2)) + assert len(locks) == 1 + assert 'PairLock' in str(locks[0]) + + # Unlock all + PairLocks.unlock_pair(pair, lock_time + timedelta(minutes=-2)) + assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50)) + + # Global lock + PairLocks.lock_pair('*', lock_time) + assert PairLocks.is_global_lock(lock_time + timedelta(minutes=-50)) + # Global lock also locks every pair seperately + assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) + assert PairLocks.is_pair_locked('XRP/USDT', lock_time + timedelta(minutes=-50)) + + if use_db: + assert len(PairLock.query.all()) > 0 + else: + # Nothing was pushed to the database + assert len(PairLock.query.all()) == 0 + # Reset use-db variable + PairLocks.reset_locks() + PairLocks.use_db = True + + +@pytest.mark.parametrize('use_db', (False, True)) +@pytest.mark.usefixtures("init_persistence") +def test_PairLocks_getlongestlock(use_db): + PairLocks.timeframe = '5m' + # No lock should be present + if use_db: + assert len(PairLock.query.all()) == 0 + else: + PairLocks.use_db = False + + assert PairLocks.use_db == use_db + + pair = 'ETH/BTC' + assert not PairLocks.is_pair_locked(pair) + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + # ETH/BTC locked for 4 minutes + assert PairLocks.is_pair_locked(pair) + lock = PairLocks.get_pair_longest_lock(pair) + + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=3) + assert lock.lock_end_time.replace(tzinfo=timezone.utc) < arrow.utcnow().shift(minutes=14) + + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=15).datetime) + assert PairLocks.is_pair_locked(pair) + + lock = PairLocks.get_pair_longest_lock(pair) + # Must be longer than above + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=14) + + PairLocks.reset_locks() + PairLocks.use_db = True diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py new file mode 100644 index 000000000..e36900a96 --- /dev/null +++ b/tests/plugins/test_protections.py @@ -0,0 +1,412 @@ +import random +from datetime import datetime, timedelta + +import pytest + +from freqtrade import constants +from freqtrade.persistence import PairLocks, Trade +from freqtrade.plugins.protectionmanager import ProtectionManager +from freqtrade.strategy.interface import SellType +from tests.conftest import get_patched_freqtradebot, log_has_re + + +def generate_mock_trade(pair: str, fee: float, is_open: bool, + sell_reason: str = SellType.SELL_SIGNAL, + min_ago_open: int = None, min_ago_close: int = None, + profit_rate: float = 0.9 + ): + open_rate = random.random() + + trade = Trade( + pair=pair, + stake_amount=0.01, + fee_open=fee, + fee_close=fee, + open_date=datetime.utcnow() - timedelta(minutes=min_ago_open or 200), + close_date=datetime.utcnow() - timedelta(minutes=min_ago_close or 30), + open_rate=open_rate, + is_open=is_open, + amount=0.01 / open_rate, + exchange='bittrex', + ) + trade.recalc_open_trade_value() + if not is_open: + trade.close(open_rate * profit_rate) + trade.sell_reason = sell_reason + + return trade + + +def test_protectionmanager(mocker, default_conf): + default_conf['protections'] = [{'method': protection} + for protection in constants.AVAILABLE_PROTECTIONS] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + for handler in freqtrade.protections._protection_handlers: + assert handler.name in constants.AVAILABLE_PROTECTIONS + if not handler.has_global_stop: + assert handler.global_stop(datetime.utcnow()) == (False, None, None) + if not handler.has_local_stop: + assert handler.stop_per_pair('XRP/BTC', datetime.utcnow()) == (False, None, None) + + +@pytest.mark.parametrize('timeframe,expected,protconf', [ + ('1m', [20, 10], + [{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 10}]), + ('5m', [100, 15], + [{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 15}]), + ('1h', [1200, 40], + [{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 40}]), + ('1d', [1440, 5], + [{"method": "StoplossGuard", "lookback_period_candles": 1, "stop_duration": 5}]), + ('1m', [20, 5], + [{"method": "StoplossGuard", "lookback_period": 20, "stop_duration_candles": 5}]), + ('5m', [15, 25], + [{"method": "StoplossGuard", "lookback_period": 15, "stop_duration_candles": 5}]), + ('1h', [50, 600], + [{"method": "StoplossGuard", "lookback_period": 50, "stop_duration_candles": 10}]), + ('1h', [60, 540], + [{"method": "StoplossGuard", "lookback_period_candles": 1, "stop_duration_candles": 9}]), +]) +def test_protections_init(mocker, default_conf, timeframe, expected, protconf): + default_conf['timeframe'] = timeframe + default_conf['protections'] = protconf + man = ProtectionManager(default_conf) + assert len(man._protection_handlers) == len(protconf) + assert man._protection_handlers[0]._lookback_period == expected[0] + assert man._protection_handlers[0]._stop_duration == expected[1] + + +@pytest.mark.usefixtures("init_persistence") +def test_stoploss_guard(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "StoplossGuard", + "lookback_period": 60, + "stop_duration": 40, + "trade_limit": 2 + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, + )) + + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + # This trade does not count, as it's closed too long ago + Trade.session.add(generate_mock_trade( + 'BCH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=250, min_ago_close=100, + )) + + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=240, min_ago_close=30, + )) + # 3 Trades closed - but the 2nd has been closed too long ago. + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'LTC/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=180, min_ago_close=30, + )) + + assert freqtrade.protections.global_stop() + assert log_has_re(message, caplog) + assert PairLocks.is_global_lock() + + # Test 5m after lock-period - this should try and relock the pair, but end-time + # should be the previous end-time + end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5) + assert freqtrade.protections.global_stop(end_time) + assert not PairLocks.is_global_lock(end_time) + + +@pytest.mark.parametrize('only_per_pair', [False, True]) +@pytest.mark.usefixtures("init_persistence") +def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair): + default_conf['protections'] = [{ + "method": "StoplossGuard", + "lookback_period": 60, + "trade_limit": 1, + "stop_duration": 60, + "only_per_pair": only_per_pair + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + pair = 'XRP/BTC' + assert not freqtrade.protections.stop_per_pair(pair) + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, profit_rate=0.9, + )) + + assert not freqtrade.protections.stop_per_pair(pair) + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + # This trade does not count, as it's closed too long ago + Trade.session.add(generate_mock_trade( + pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=250, min_ago_close=100, profit_rate=0.9, + )) + # Trade does not count for per pair stop as it's the wrong pair. + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=240, min_ago_close=30, profit_rate=0.9, + )) + # 3 Trades closed - but the 2nd has been closed too long ago. + assert not freqtrade.protections.stop_per_pair(pair) + assert freqtrade.protections.global_stop() != only_per_pair + if not only_per_pair: + assert log_has_re(message, caplog) + else: + assert not log_has_re(message, caplog) + + caplog.clear() + + # 2nd Trade that counts with correct pair + Trade.session.add(generate_mock_trade( + pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=180, min_ago_close=30, profit_rate=0.9, + )) + + assert freqtrade.protections.stop_per_pair(pair) + assert freqtrade.protections.global_stop() != only_per_pair + assert PairLocks.is_pair_locked(pair) + assert PairLocks.is_global_lock() != only_per_pair + + +@pytest.mark.usefixtures("init_persistence") +def test_CooldownPeriod(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "CooldownPeriod", + "stop_duration": 60, + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, + )) + + assert not freqtrade.protections.global_stop() + assert freqtrade.protections.stop_per_pair('XRP/BTC') + assert PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=205, min_ago_close=35, + )) + + assert not freqtrade.protections.global_stop() + assert not PairLocks.is_pair_locked('ETH/BTC') + assert freqtrade.protections.stop_per_pair('ETH/BTC') + assert PairLocks.is_pair_locked('ETH/BTC') + assert not PairLocks.is_global_lock() + + +@pytest.mark.usefixtures("init_persistence") +def test_LowProfitPairs(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "LowProfitPairs", + "lookback_period": 400, + "stop_duration": 60, + "trade_limit": 2, + "required_profit": 0.0, + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=800, min_ago_close=450, profit_rate=0.9, + )) + + # Not locked with 1 trade + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=120, profit_rate=0.9, + )) + + # Not locked with 1 trade (first trade is outside of lookback_period) + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + # Add positive trade + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=20, min_ago_close=10, profit_rate=1.15, + )) + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=110, min_ago_close=20, profit_rate=0.8, + )) + + # Locks due to 2nd trade + assert not freqtrade.protections.global_stop() + assert freqtrade.protections.stop_per_pair('XRP/BTC') + assert PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + +@pytest.mark.usefixtures("init_persistence") +def test_MaxDrawdown(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "MaxDrawdown", + "lookback_period": 1000, + "stop_duration": 60, + "trade_limit": 3, + "max_allowed_drawdown": 0.15 + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to Max.*" + + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1000, min_ago_close=900, profit_rate=1.1, + )) + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1000, min_ago_close=900, profit_rate=1.1, + )) + Trade.session.add(generate_mock_trade( + 'NEO/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1000, min_ago_close=900, profit_rate=1.1, + )) + # No losing trade yet ... so max_drawdown will raise exception + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=500, min_ago_close=400, profit_rate=0.9, + )) + # Not locked with one trade + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1200, min_ago_close=1100, profit_rate=0.5, + )) + + # Not locked with 1 trade (2nd trade is outside of lookback_period) + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + assert not log_has_re(message, caplog) + + # Winning trade ... (should not lock, does not change drawdown!) + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=320, min_ago_close=410, profit_rate=1.5, + )) + assert not freqtrade.protections.global_stop() + assert not PairLocks.is_global_lock() + + caplog.clear() + + # Add additional negative trade, causing a loss of > 15% + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=20, min_ago_close=10, profit_rate=0.8, + )) + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + # local lock not supported + assert not PairLocks.is_pair_locked('XRP/BTC') + assert freqtrade.protections.global_stop() + assert PairLocks.is_global_lock() + assert log_has_re(message, caplog) + + +@pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ + ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2, "stop_duration": 60}, + "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " + "2 stoplosses within 60 minutes.'}]", + None + ), + ({"method": "CooldownPeriod", "stop_duration": 60}, + "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 60 minutes.'}]", + None + ), + ({"method": "LowProfitPairs", "lookback_period": 60, "stop_duration": 60}, + "[{'LowProfitPairs': 'LowProfitPairs - Low Profit Protection, locks pairs with " + "profit < 0.0 within 60 minutes.'}]", + None + ), + ({"method": "MaxDrawdown", "lookback_period": 60, "stop_duration": 60}, + "[{'MaxDrawdown': 'MaxDrawdown - Max drawdown protection, stop trading if drawdown is > 0.0 " + "within 60 minutes.'}]", + None + ), + ({"method": "StoplossGuard", "lookback_period_candles": 12, "trade_limit": 2, + "stop_duration": 60}, + "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " + "2 stoplosses within 12 candles.'}]", + None + ), + ({"method": "CooldownPeriod", "stop_duration_candles": 5}, + "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 5 candles.'}]", + None + ), + ({"method": "LowProfitPairs", "lookback_period_candles": 11, "stop_duration": 60}, + "[{'LowProfitPairs': 'LowProfitPairs - Low Profit Protection, locks pairs with " + "profit < 0.0 within 11 candles.'}]", + None + ), + ({"method": "MaxDrawdown", "lookback_period_candles": 20, "stop_duration": 60}, + "[{'MaxDrawdown': 'MaxDrawdown - Max drawdown protection, stop trading if drawdown is > 0.0 " + "within 20 candles.'}]", + None + ), +]) +def test_protection_manager_desc(mocker, default_conf, protectionconf, + desc_expected, exception_expected): + + default_conf['protections'] = [protectionconf] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + short_desc = str(freqtrade.protections.short_desc()) + assert short_desc == desc_expected diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index ef1c1bc16..4b36f4b4e 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -8,12 +8,12 @@ import pytest from numpy import isnan from freqtrade.edge import PairInfo -from freqtrade.exceptions import DependencyException, TemporaryError +from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State -from tests.conftest import get_patched_freqtradebot, patch_get_signal, create_mock_trades +from tests.conftest import create_mock_trades, get_patched_freqtradebot, patch_get_signal # Functions for recurrent object patching @@ -62,14 +62,14 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'fee_close_cost': ANY, 'fee_close_currency': ANY, 'open_rate_requested': ANY, - 'open_trade_price': 0.0010025, + 'open_trade_value': 0.0010025, 'close_rate_requested': ANY, 'sell_reason': ANY, 'sell_order_status': ANY, 'min_rate': ANY, 'max_rate': ANY, 'strategy': ANY, - 'ticker_interval': ANY, + 'timeframe': 5, 'open_order_id': ANY, 'close_date': None, 'close_date_hum': None, @@ -77,7 +77,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_rate': 1.098e-05, 'close_rate': None, 'current_rate': 1.099e-05, - 'amount': 91.07468124, + 'amount': 91.07468123, + 'amount_requested': 91.07468123, 'stake_amount': 0.001, 'close_profit': None, 'close_profit_pct': None, @@ -85,19 +86,21 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'current_profit': -0.00408133, 'current_profit_pct': -0.41, 'current_profit_abs': -4.09e-06, - 'stop_loss': 9.882e-06, + 'profit_ratio': -0.00408133, + 'profit_pct': -0.41, + 'profit_abs': -4.09e-06, 'stop_loss_abs': 9.882e-06, 'stop_loss_pct': -10.0, 'stop_loss_ratio': -0.1, 'stoploss_order_id': None, 'stoploss_last_update': ANY, 'stoploss_last_update_timestamp': ANY, - 'initial_stop_loss': 9.882e-06, 'initial_stop_loss_abs': 9.882e-06, 'initial_stop_loss_pct': -10.0, 'initial_stop_loss_ratio': -0.1, 'stoploss_current_dist': -1.1080000000000002e-06, 'stoploss_current_dist_ratio': -0.10081893, + 'stoploss_current_dist_pct': -10.08, 'stoploss_entry_dist': -0.00010475, 'stoploss_entry_dist_ratio': -0.10448878, 'open_order': None, @@ -105,7 +108,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: } mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', - MagicMock(side_effect=DependencyException("Pair 'ETH/BTC' not available"))) + MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) assert isnan(results[0]['current_rate']) @@ -124,14 +127,14 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'fee_close_cost': ANY, 'fee_close_currency': ANY, 'open_rate_requested': ANY, - 'open_trade_price': ANY, + 'open_trade_value': ANY, 'close_rate_requested': ANY, 'sell_reason': ANY, 'sell_order_status': ANY, 'min_rate': ANY, 'max_rate': ANY, 'strategy': ANY, - 'ticker_interval': ANY, + 'timeframe': ANY, 'open_order_id': ANY, 'close_date': None, 'close_date_hum': None, @@ -139,7 +142,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_rate': 1.098e-05, 'close_rate': None, 'current_rate': ANY, - 'amount': 91.07468124, + 'amount': 91.07468123, + 'amount_requested': 91.07468123, 'stake_amount': 0.001, 'close_profit': None, 'close_profit_pct': None, @@ -147,19 +151,21 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'current_profit': ANY, 'current_profit_pct': ANY, 'current_profit_abs': ANY, - 'stop_loss': 9.882e-06, + 'profit_ratio': ANY, + 'profit_pct': ANY, + 'profit_abs': ANY, 'stop_loss_abs': 9.882e-06, 'stop_loss_pct': -10.0, 'stop_loss_ratio': -0.1, 'stoploss_order_id': None, 'stoploss_last_update': ANY, 'stoploss_last_update_timestamp': ANY, - 'initial_stop_loss': 9.882e-06, 'initial_stop_loss_abs': 9.882e-06, 'initial_stop_loss_pct': -10.0, 'initial_stop_loss_ratio': -0.1, 'stoploss_current_dist': ANY, 'stoploss_current_dist_ratio': ANY, + 'stoploss_current_dist_pct': ANY, 'stoploss_entry_dist': -0.00010475, 'stoploss_entry_dist_ratio': -0.10448878, 'open_order': None, @@ -207,7 +213,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert '-0.41% (-0.06)' == result[0][3] mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', - MagicMock(side_effect=DependencyException("Pair 'ETH/BTC' not available"))) + MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] assert 'ETH/BTC' in result[0][1] @@ -250,11 +256,11 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, assert days['fiat_display_currency'] == default_conf['fiat_display_currency'] for day in days['data']: # [datetime.date(2018, 1, 11), '0.00000000 BTC', '0.000 USD'] - assert (day['abs_profit'] == '0.00000000' or - day['abs_profit'] == '0.00006217') + assert (day['abs_profit'] == 0.0 or + day['abs_profit'] == 0.00006217) - assert (day['fiat_value'] == '0.000' or - day['fiat_value'] == '0.767') + assert (day['fiat_value'] == 0.0 or + day['fiat_value'] == 0.76748865) # ensure first day is current date assert str(days['data'][0]['date']) == str(datetime.utcnow().date()) @@ -281,12 +287,65 @@ def test_rpc_trade_history(mocker, default_conf, markets, fee): assert isinstance(trades['trades'][1], dict) trades = rpc._rpc_trade_history(0) - assert len(trades['trades']) == 3 - assert trades['trades_count'] == 3 - # The first trade is for ETH ... sorting is descending - assert trades['trades'][-1]['pair'] == 'ETH/BTC' - assert trades['trades'][0]['pair'] == 'ETC/BTC' - assert trades['trades'][1]['pair'] == 'ETC/BTC' + assert len(trades['trades']) == 2 + assert trades['trades_count'] == 2 + # The first closed trade is for ETC ... sorting is descending + assert trades['trades'][-1]['pair'] == 'ETC/BTC' + assert trades['trades'][0]['pair'] == 'XRP/BTC' + + +def test_rpc_delete_trade(mocker, default_conf, fee, markets, caplog): + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + stoploss_mock = MagicMock() + cancel_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + cancel_order=cancel_mock, + cancel_stoploss_order=stoploss_mock, + ) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + freqtradebot.strategy.order_types['stoploss_on_exchange'] = True + create_mock_trades(fee) + rpc = RPC(freqtradebot) + with pytest.raises(RPCException, match='invalid argument'): + rpc._rpc_delete('200') + + trades = Trade.query.all() + trades[1].stoploss_order_id = '1234' + trades[2].stoploss_order_id = '1234' + assert len(trades) > 2 + + res = rpc._rpc_delete('1') + assert isinstance(res, dict) + assert res['result'] == 'success' + assert res['trade_id'] == '1' + assert res['cancel_order_count'] == 1 + assert cancel_mock.call_count == 1 + assert stoploss_mock.call_count == 0 + cancel_mock.reset_mock() + stoploss_mock.reset_mock() + + res = rpc._rpc_delete('2') + assert isinstance(res, dict) + assert cancel_mock.call_count == 1 + assert stoploss_mock.call_count == 1 + assert res['cancel_order_count'] == 2 + + stoploss_mock = mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', + side_effect=InvalidOrderException) + + res = rpc._rpc_delete('3') + assert stoploss_mock.call_count == 1 + stoploss_mock.reset_mock() + + cancel_mock = mocker.patch('freqtrade.exchange.Exchange.cancel_order', + side_effect=InvalidOrderException) + + res = rpc._rpc_delete('4') + assert cancel_mock.call_count == 1 + assert stoploss_mock.call_count == 0 def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, @@ -363,7 +422,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Test non-available pair mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', - MagicMock(side_effect=DependencyException("Pair 'ETH/BTC' not available"))) + MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' @@ -592,7 +651,7 @@ def test_rpc_stopbuy(mocker, default_conf) -> None: assert freqtradebot.config['max_open_trades'] != 0 result = rpc._rpc_stopbuy() - assert {'status': 'No more buy will occur from now. Run /reload_conf to reset.'} == result + assert {'status': 'No more buy will occur from now. Run /reload_config to reset.'} == result assert freqtradebot.config['max_open_trades'] == 0 @@ -604,11 +663,12 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker, cancel_order=cancel_order_mock, - get_order=MagicMock( + fetch_order=MagicMock( return_value={ 'status': 'closed', 'type': 'limit', - 'side': 'buy' + 'side': 'buy', + 'filled': 0.0, } ), get_fee=fee, @@ -634,6 +694,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: msg = rpc._rpc_forcesell('all') assert msg == {'result': 'Created sell orders for all open trades.'} + freqtradebot.enter_positions() msg = rpc._rpc_forcesell('1') assert msg == {'result': 'Created sell order for trade 1.'} @@ -646,17 +707,26 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING assert cancel_order_mock.call_count == 0 + freqtradebot.enter_positions() # make an limit-buy open trade trade = Trade.query.filter(Trade.id == '1').first() filled_amount = trade.amount / 2 + # Fetch order - it's open first, and closed after cancel_order is called. mocker.patch( - 'freqtrade.exchange.Exchange.get_order', - return_value={ + 'freqtrade.exchange.Exchange.fetch_order', + side_effect=[{ + 'id': '1234', 'status': 'open', 'type': 'limit', 'side': 'buy', 'filled': filled_amount - } + }, { + 'id': '1234', + 'status': 'closed', + 'type': 'limit', + 'side': 'buy', + 'filled': filled_amount + }] ) # check that the trade is called, which is done by ensuring exchange.cancel_order is called # and trade amount is updated @@ -664,12 +734,22 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: assert cancel_order_mock.call_count == 1 assert trade.amount == filled_amount + mocker.patch( + 'freqtrade.exchange.Exchange.fetch_order', + return_value={ + 'status': 'open', + 'type': 'limit', + 'side': 'buy', + 'filled': filled_amount + }) + + freqtradebot.config['max_open_trades'] = 3 freqtradebot.enter_positions() trade = Trade.query.filter(Trade.id == '2').first() amount = trade.amount # make an limit-buy open trade, if there is no 'filled', don't sell it mocker.patch( - 'freqtrade.exchange.Exchange.get_order', + 'freqtrade.exchange.Exchange.fetch_order', return_value={ 'status': 'open', 'type': 'limit', @@ -683,20 +763,22 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: assert cancel_order_mock.call_count == 2 assert trade.amount == amount - freqtradebot.enter_positions() # make an limit-sell open trade mocker.patch( - 'freqtrade.exchange.Exchange.get_order', + 'freqtrade.exchange.Exchange.fetch_order', return_value={ 'status': 'open', 'type': 'limit', - 'side': 'sell' + 'side': 'sell', + 'amount': amount, + 'remaining': amount, + 'filled': 0.0 } ) msg = rpc._rpc_forcesell('3') assert msg == {'result': 'Created sell order for trade 3.'} # status quo, no exchange calls - assert cancel_order_mock.call_count == 2 + assert cancel_order_mock.call_count == 3 def test_performance_handle(default_conf, ticker, limit_buy_order, fee, @@ -755,10 +837,10 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: assert counts["current"] == 1 -def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None: +def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order_open) -> None: default_conf['forcebuy_enable'] = True mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) + buy_mm = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), @@ -786,7 +868,8 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None assert trade.open_rate == 0.0001 # Test buy pair not with stakes - with pytest.raises(RPCException, match=r'Wrong pair selected. Please pairs with stake.*'): + with pytest.raises(RPCException, + match=r'Wrong pair selected. Only pairs with stake-currency.*'): rpc._rpc_forcebuy('LTC/ETH', 0.0001) pair = 'XRP/BTC' diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 9b247fefc..a1f4f7c9d 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -2,7 +2,8 @@ Unit test file for rpc/api_server.py """ -from datetime import datetime +from datetime import datetime, timedelta, timezone +from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock import pytest @@ -10,10 +11,12 @@ from flask import Flask from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ -from freqtrade.persistence import Trade +from freqtrade.loggers import setup_logging, setup_logging_pre +from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc.api_server import BASE_URI, ApiServer -from freqtrade.state import State -from tests.conftest import get_patched_freqtradebot, log_has, patch_get_signal, create_mock_trades +from freqtrade.state import RunMode, State +from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, patch_get_signal + _TEST_USER = "FreqTrader" _TEST_PASS = "SuperSecurePassword1!" @@ -21,9 +24,13 @@ _TEST_PASS = "SuperSecurePassword1!" @pytest.fixture def botclient(default_conf, mocker): + setup_logging_pre() + setup_logging(default_conf) + default_conf['runmode'] = RunMode.DRY_RUN default_conf.update({"api_server": {"enabled": True, "listen_ip_address": "127.0.0.1", "listen_port": 8080, + "CORS_origins": ['http://example.com'], "username": _TEST_USER, "password": _TEST_PASS, }}) @@ -40,13 +47,19 @@ def client_post(client, url, data={}): content_type="application/json", data=data, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS), - 'Origin': 'example.com'}) + 'Origin': 'http://example.com'}) def client_get(client, url): # Add fake Origin to ensure CORS kicks in return client.get(url, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS), - 'Origin': 'example.com'}) + 'Origin': 'http://example.com'}) + + +def client_delete(client, url): + # Add fake Origin to ensure CORS kicks in + return client.delete(url, headers={'Authorization': _basic_auth_str(_TEST_USER, _TEST_PASS), + 'Origin': 'http://example.com'}) def assert_response(response, expected_code=200, needs_cors=True): @@ -54,6 +67,7 @@ def assert_response(response, expected_code=200, needs_cors=True): assert response.content_type == "application/json" if needs_cors: assert ('Access-Control-Allow-Credentials', 'true') in response.headers._list + assert ('Access-Control-Allow-Origin', 'http://example.com') in response.headers._list def test_api_not_found(botclient): @@ -79,20 +93,20 @@ def test_api_unauthorized(botclient): assert rc.json == {'error': 'Unauthorized'} # Change only username - ftbot.config['api_server']['username'] = "Ftrader" + ftbot.config['api_server']['username'] = 'Ftrader' rc = client_get(client, f"{BASE_URI}/version") assert_response(rc, 401) assert rc.json == {'error': 'Unauthorized'} # Change only password ftbot.config['api_server']['username'] = _TEST_USER - ftbot.config['api_server']['password'] = "WrongPassword" + ftbot.config['api_server']['password'] = 'WrongPassword' rc = client_get(client, f"{BASE_URI}/version") assert_response(rc, 401) assert rc.json == {'error': 'Unauthorized'} - ftbot.config['api_server']['username'] = "Ftrader" - ftbot.config['api_server']['password'] = "WrongPassword" + ftbot.config['api_server']['username'] = 'Ftrader' + ftbot.config['api_server']['password'] = 'WrongPassword' rc = client_get(client, f"{BASE_URI}/version") assert_response(rc, 401) @@ -110,7 +124,7 @@ def test_api_token_login(botclient): rc = client.get(f"{BASE_URI}/count", content_type="application/json", headers={'Authorization': f'Bearer {rc.json["access_token"]}', - 'Origin': 'example.com'}) + 'Origin': 'http://example.com'}) assert_response(rc) @@ -122,7 +136,7 @@ def test_api_token_refresh(botclient): content_type="application/json", data=None, headers={'Authorization': f'Bearer {rc.json["refresh_token"]}', - 'Origin': 'example.com'}) + 'Origin': 'http://example.com'}) assert_response(rc) assert 'access_token' in rc.json assert 'refresh_token' not in rc.json @@ -251,10 +265,10 @@ def test_api_cleanup(default_conf, mocker, caplog): def test_api_reloadconf(botclient): ftbot, client = botclient - rc = client_post(client, f"{BASE_URI}/reload_conf") + rc = client_post(client, f"{BASE_URI}/reload_config") assert_response(rc) - assert rc.json == {'status': 'reloading config ...'} - assert ftbot.state == State.RELOAD_CONF + assert rc.json == {'status': 'Reloading config ...'} + assert ftbot.state == State.RELOAD_CONFIG def test_api_stopbuy(botclient): @@ -263,7 +277,7 @@ def test_api_stopbuy(botclient): rc = client_post(client, f"{BASE_URI}/stopbuy") assert_response(rc) - assert rc.json == {'status': 'No more buy will occur from now. Run /reload_conf to reset.'} + assert rc.json == {'status': 'No more buy will occur from now. Run /reload_config to reset.'} assert ftbot.config['max_open_trades'] == 0 @@ -314,6 +328,30 @@ def test_api_count(botclient, mocker, ticker, fee, markets): assert rc.json["max"] == 1.0 +def test_api_locks(botclient): + ftbot, client = botclient + + rc = client_get(client, f"{BASE_URI}/locks") + assert_response(rc) + + assert 'locks' in rc.json + + assert rc.json['lock_count'] == 0 + assert rc.json['lock_count'] == len(rc.json['locks']) + + PairLocks.lock_pair('ETH/BTC', datetime.now(timezone.utc) + timedelta(minutes=4), 'randreason') + PairLocks.lock_pair('XRP/BTC', datetime.now(timezone.utc) + timedelta(minutes=20), 'deadbeef') + + rc = client_get(client, f"{BASE_URI}/locks") + assert_response(rc) + + assert rc.json['lock_count'] == 2 + assert rc.json['lock_count'] == len(rc.json['locks']) + assert 'ETH/BTC' in (rc.json['locks'][0]['pair'], rc.json['locks'][1]['pair']) + assert 'randreason' in (rc.json['locks'][0]['reason'], rc.json['locks'][1]['reason']) + assert 'deadbeef' in (rc.json['locks'][0]['reason'], rc.json['locks'][1]['reason']) + + def test_api_show_config(botclient, mocker): ftbot, client = botclient patch_get_signal(ftbot, (True, False)) @@ -322,7 +360,9 @@ def test_api_show_config(botclient, mocker): assert_response(rc) assert 'dry_run' in rc.json assert rc.json['exchange'] == 'bittrex' - assert rc.json['ticker_interval'] == '5m' + assert rc.json['timeframe'] == '5m' + assert rc.json['timeframe_ms'] == 300000 + assert rc.json['timeframe_min'] == 5 assert rc.json['state'] == 'running' assert not rc.json['trailing_stop'] assert 'bid_strategy' in rc.json @@ -347,7 +387,7 @@ def test_api_daily(botclient, mocker, ticker, fee, markets): assert rc.json['data'][0]['date'] == str(datetime.utcnow().date()) -def test_api_trades(botclient, mocker, ticker, fee, markets): +def test_api_trades(botclient, mocker, fee, markets): ftbot, client = botclient patch_get_signal(ftbot, (True, False)) mocker.patch.multiple( @@ -363,12 +403,81 @@ def test_api_trades(botclient, mocker, ticker, fee, markets): rc = client_get(client, f"{BASE_URI}/trades") assert_response(rc) - assert len(rc.json['trades']) == 3 - assert rc.json['trades_count'] == 3 - rc = client_get(client, f"{BASE_URI}/trades?limit=2") - assert_response(rc) assert len(rc.json['trades']) == 2 assert rc.json['trades_count'] == 2 + rc = client_get(client, f"{BASE_URI}/trades?limit=1") + assert_response(rc) + assert len(rc.json['trades']) == 1 + assert rc.json['trades_count'] == 1 + + +def test_api_delete_trade(botclient, mocker, fee, markets): + ftbot, client = botclient + patch_get_signal(ftbot, (True, False)) + stoploss_mock = MagicMock() + cancel_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + cancel_order=cancel_mock, + cancel_stoploss_order=stoploss_mock, + ) + rc = client_delete(client, f"{BASE_URI}/trades/1") + # Error - trade won't exist yet. + assert_response(rc, 502) + + create_mock_trades(fee) + ftbot.strategy.order_types['stoploss_on_exchange'] = True + trades = Trade.query.all() + trades[1].stoploss_order_id = '1234' + assert len(trades) > 2 + + rc = client_delete(client, f"{BASE_URI}/trades/1") + assert_response(rc) + assert rc.json['result_msg'] == 'Deleted trade 1. Closed 1 open orders.' + assert len(trades) - 1 == len(Trade.query.all()) + assert cancel_mock.call_count == 1 + + cancel_mock.reset_mock() + rc = client_delete(client, f"{BASE_URI}/trades/1") + # Trade is gone now. + assert_response(rc, 502) + assert cancel_mock.call_count == 0 + + assert len(trades) - 1 == len(Trade.query.all()) + rc = client_delete(client, f"{BASE_URI}/trades/2") + assert_response(rc) + assert rc.json['result_msg'] == 'Deleted trade 2. Closed 2 open orders.' + assert len(trades) - 2 == len(Trade.query.all()) + assert stoploss_mock.call_count == 1 + + +def test_api_logs(botclient): + ftbot, client = botclient + rc = client_get(client, f"{BASE_URI}/logs") + assert_response(rc) + assert len(rc.json) == 2 + assert 'logs' in rc.json + # Using a fixed comparison here would make this test fail! + assert rc.json['log_count'] > 1 + assert len(rc.json['logs']) == rc.json['log_count'] + + assert isinstance(rc.json['logs'][0], list) + # date + assert isinstance(rc.json['logs'][0][0], str) + # created_timestamp + assert isinstance(rc.json['logs'][0][1], float) + assert isinstance(rc.json['logs'][0][2], str) + assert isinstance(rc.json['logs'][0][3], str) + assert isinstance(rc.json['logs'][0][4], str) + + rc = client_get(client, f"{BASE_URI}/logs?limit=5") + assert_response(rc) + assert len(rc.json) == 2 + assert 'logs' in rc.json + # Using a fixed comparison here would make this test fail! + assert rc.json['log_count'] == 5 + assert len(rc.json['logs']) == rc.json['log_count'] def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): @@ -386,6 +495,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): assert rc.json == {"error": "Error querying _edge: Edge is not enabled."} +@pytest.mark.usefixtures("init_persistence") def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, limit_sell_order): ftbot, client = botclient patch_get_signal(ftbot, (True, False)) @@ -413,6 +523,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li assert rc.json['best_pair'] == '' assert rc.json['best_rate'] == 0 + trade = Trade.query.first() trade.update(limit_sell_order) trade.close_date = datetime.utcnow() @@ -428,14 +539,14 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li 'latest_trade_date': 'just now', 'latest_trade_timestamp': ANY, 'profit_all_coin': 6.217e-05, - 'profit_all_fiat': 0, + 'profit_all_fiat': 0.76748865, 'profit_all_percent': 6.2, 'profit_all_percent_mean': 6.2, 'profit_all_ratio_mean': 0.06201058, 'profit_all_percent_sum': 6.2, 'profit_all_ratio_sum': 0.06201058, 'profit_closed_coin': 6.217e-05, - 'profit_closed_fiat': 0, + 'profit_closed_fiat': 0.76748865, 'profit_closed_percent': 6.2, 'profit_closed_ratio_mean': 0.06201058, 'profit_closed_percent_mean': 6.2, @@ -443,9 +554,40 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li 'profit_closed_percent_sum': 6.2, 'trade_count': 1, 'closed_trade_count': 1, + 'winning_trades': 1, + 'losing_trades': 0, } +@pytest.mark.usefixtures("init_persistence") +def test_api_stats(botclient, mocker, ticker, fee, markets,): + ftbot, client = botclient + patch_get_signal(ftbot, (True, False)) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value=ticker), + fetch_ticker=ticker, + get_fee=fee, + markets=PropertyMock(return_value=markets) + ) + + rc = client_get(client, f"{BASE_URI}/stats") + assert_response(rc, 200) + assert 'durations' in rc.json + assert 'sell_reasons' in rc.json + + create_mock_trades(fee) + + rc = client_get(client, f"{BASE_URI}/stats") + assert_response(rc, 200) + assert 'durations' in rc.json + assert 'sell_reasons' in rc.json + + assert 'wins' in rc.json['durations'] + assert 'losses' in rc.json['durations'] + assert 'draws' in rc.json['durations'] + + def test_api_performance(botclient, mocker, ticker, fee): ftbot, client = botclient patch_get_signal(ftbot, (True, False)) @@ -512,7 +654,8 @@ def test_api_status(botclient, mocker, ticker, fee, markets): rc = client_get(client, f"{BASE_URI}/status") assert_response(rc) assert len(rc.json) == 1 - assert rc.json == [{'amount': 91.07468124, + assert rc.json == [{'amount': 91.07468123, + 'amount_requested': 91.07468123, 'base_currency': 'BTC', 'close_date': None, 'close_date_hum': None, @@ -524,6 +667,9 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'current_profit': -0.00408133, 'current_profit_pct': -0.41, 'current_profit_abs': -4.09e-06, + 'profit_ratio': -0.00408133, + 'profit_pct': -0.41, + 'profit_abs': -4.09e-06, 'current_rate': 1.099e-05, 'open_date': ANY, 'open_date_hum': 'just now', @@ -532,19 +678,18 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'open_rate': 1.098e-05, 'pair': 'ETH/BTC', 'stake_amount': 0.001, - 'stop_loss': 9.882e-06, 'stop_loss_abs': 9.882e-06, 'stop_loss_pct': -10.0, 'stop_loss_ratio': -0.1, 'stoploss_order_id': None, 'stoploss_last_update': ANY, 'stoploss_last_update_timestamp': ANY, - 'initial_stop_loss': 9.882e-06, 'initial_stop_loss_abs': 9.882e-06, 'initial_stop_loss_pct': -10.0, 'initial_stop_loss_ratio': -0.1, 'stoploss_current_dist': -1.1080000000000002e-06, 'stoploss_current_dist_ratio': -0.10081893, + 'stoploss_current_dist_pct': -10.08, 'stoploss_entry_dist': -0.00010475, 'stoploss_entry_dist_ratio': -0.10448878, 'trade_id': 1, @@ -562,11 +707,11 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'min_rate': 1.098e-05, 'open_order_id': None, 'open_rate_requested': 1.098e-05, - 'open_trade_price': 0.0010025, + 'open_trade_value': 0.0010025, 'sell_reason': None, 'sell_order_status': None, 'strategy': 'DefaultStrategy', - 'ticker_interval': 5, + 'timeframe': 5, 'exchange': 'bittrex', }] @@ -620,7 +765,7 @@ def test_api_forcebuy(botclient, mocker, fee): assert rc.json == {"error": "Error querying _forcebuy: Forcebuy not enabled."} # enable forcebuy - ftbot.config["forcebuy_enable"] = True + ftbot.config['forcebuy_enable'] = True fbuy_mock = MagicMock(return_value=None) mocker.patch("freqtrade.rpc.RPC._rpc_forcebuy", fbuy_mock) @@ -633,6 +778,7 @@ def test_api_forcebuy(botclient, mocker, fee): fbuy_mock = MagicMock(return_value=Trade( pair='ETH/ETH', amount=1, + amount_requested=1, exchange='bittrex', stake_amount=1, open_rate=0.245441, @@ -649,6 +795,7 @@ def test_api_forcebuy(botclient, mocker, fee): data='{"pair": "ETH/BTC"}') assert_response(rc) assert rc.json == {'amount': 1, + 'amount_requested': 1, 'trade_id': None, 'close_date': None, 'close_date_hum': None, @@ -660,20 +807,22 @@ def test_api_forcebuy(botclient, mocker, fee): 'open_rate': 0.245441, 'pair': 'ETH/ETH', 'stake_amount': 1, - 'stop_loss': None, 'stop_loss_abs': None, 'stop_loss_pct': None, 'stop_loss_ratio': None, 'stoploss_order_id': None, 'stoploss_last_update': None, 'stoploss_last_update_timestamp': None, - 'initial_stop_loss': None, 'initial_stop_loss_abs': None, 'initial_stop_loss_pct': None, 'initial_stop_loss_ratio': None, 'close_profit': None, + 'close_profit_pct': None, 'close_profit_abs': None, 'close_rate_requested': None, + 'profit_ratio': None, + 'profit_pct': None, + 'profit_abs': None, 'fee_close': 0.0025, 'fee_close_cost': None, 'fee_close_currency': None, @@ -685,11 +834,11 @@ def test_api_forcebuy(botclient, mocker, fee): 'min_rate': None, 'open_order_id': '123456', 'open_rate_requested': None, - 'open_trade_price': 0.2460546025, + 'open_trade_value': 0.24605460, 'sell_reason': None, 'sell_order_status': None, 'strategy': None, - 'ticker_interval': None, + 'timeframe': None, 'exchange': 'bittrex', } @@ -716,3 +865,179 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): data='{"tradeid": "1"}') assert_response(rc) assert rc.json == {'result': 'Created sell order for trade 1.'} + + +def test_api_pair_candles(botclient, ohlcv_history): + ftbot, client = botclient + timeframe = '5m' + amount = 3 + + # No pair + rc = client_get(client, + f"{BASE_URI}/pair_candles?limit={amount}&timeframe={timeframe}") + assert_response(rc, 400) + + # No timeframe + rc = client_get(client, + f"{BASE_URI}/pair_candles?pair=XRP%2FBTC") + assert_response(rc, 400) + + rc = client_get(client, + f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") + assert_response(rc) + assert 'columns' in rc.json + assert 'data_start_ts' in rc.json + assert 'data_start' in rc.json + assert 'data_stop' in rc.json + assert 'data_stop_ts' in rc.json + assert len(rc.json['data']) == 0 + ohlcv_history['sma'] = ohlcv_history['close'].rolling(2).mean() + ohlcv_history['buy'] = 0 + ohlcv_history.loc[1, 'buy'] = 1 + ohlcv_history['sell'] = 0 + + ftbot.dataprovider._set_cached_df("XRP/BTC", timeframe, ohlcv_history) + + rc = client_get(client, + f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") + assert_response(rc) + assert 'strategy' in rc.json + assert rc.json['strategy'] == 'DefaultStrategy' + assert 'columns' in rc.json + assert 'data_start_ts' in rc.json + assert 'data_start' in rc.json + assert 'data_stop' in rc.json + assert 'data_stop_ts' in rc.json + assert rc.json['data_start'] == '2017-11-26 08:50:00+00:00' + assert rc.json['data_start_ts'] == 1511686200000 + assert rc.json['data_stop'] == '2017-11-26 09:00:00+00:00' + assert rc.json['data_stop_ts'] == 1511686800000 + assert isinstance(rc.json['columns'], list) + assert rc.json['columns'] == ['date', 'open', 'high', + 'low', 'close', 'volume', 'sma', 'buy', 'sell', + '__date_ts', '_buy_signal_open', '_sell_signal_open'] + assert 'pair' in rc.json + assert rc.json['pair'] == 'XRP/BTC' + + assert 'data' in rc.json + assert len(rc.json['data']) == amount + + assert (rc.json['data'] == + [['2017-11-26 08:50:00', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, + None, 0, 0, 1511686200000, None, None], + ['2017-11-26 08:55:00', 8.88e-05, 8.942e-05, 8.88e-05, + 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 1511686500000, 8.88e-05, None], + ['2017-11-26 09:00:00', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, + 0.7039405, 8.885000000000002e-05, 0, 0, 1511686800000, None, None] + + ]) + + +def test_api_pair_history(botclient, ohlcv_history): + ftbot, client = botclient + timeframe = '5m' + + # No pair + rc = client_get(client, + f"{BASE_URI}/pair_history?timeframe={timeframe}" + "&timerange=20180111-20180112&strategy=DefaultStrategy") + assert_response(rc, 400) + + # No Timeframe + rc = client_get(client, + f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC" + "&timerange=20180111-20180112&strategy=DefaultStrategy") + assert_response(rc, 400) + + # No timerange + rc = client_get(client, + f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" + "&strategy=DefaultStrategy") + assert_response(rc, 400) + + # No strategy + rc = client_get(client, + f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" + "&timerange=20180111-20180112") + assert_response(rc, 400) + + # Working + rc = client_get(client, + f"{BASE_URI}/pair_history?pair=UNITTEST%2FBTC&timeframe={timeframe}" + "&timerange=20180111-20180112&strategy=DefaultStrategy") + assert_response(rc, 200) + assert rc.json['length'] == 289 + assert len(rc.json['data']) == rc.json['length'] + assert 'columns' in rc.json + assert 'data' in rc.json + assert rc.json['pair'] == 'UNITTEST/BTC' + assert rc.json['strategy'] == 'DefaultStrategy' + assert rc.json['data_start'] == '2018-01-11 00:00:00+00:00' + assert rc.json['data_start_ts'] == 1515628800000 + assert rc.json['data_stop'] == '2018-01-12 00:00:00+00:00' + assert rc.json['data_stop_ts'] == 1515715200000 + + +def test_api_plot_config(botclient): + ftbot, client = botclient + + rc = client_get(client, f"{BASE_URI}/plot_config") + assert_response(rc) + assert rc.json == {} + + ftbot.strategy.plot_config = {'main_plot': {'sma': {}}, + 'subplots': {'RSI': {'rsi': {'color': 'red'}}}} + rc = client_get(client, f"{BASE_URI}/plot_config") + assert_response(rc) + assert rc.json == ftbot.strategy.plot_config + assert isinstance(rc.json['main_plot'], dict) + + +def test_api_strategies(botclient): + ftbot, client = botclient + + rc = client_get(client, f"{BASE_URI}/strategies") + + assert_response(rc) + assert rc.json == {'strategies': ['DefaultStrategy', 'TestStrategyLegacy']} + + +def test_api_strategy(botclient): + ftbot, client = botclient + + rc = client_get(client, f"{BASE_URI}/strategy/DefaultStrategy") + + assert_response(rc) + assert rc.json['strategy'] == 'DefaultStrategy' + + data = (Path(__file__).parents[1] / "strategy/strats/default_strategy.py").read_text() + assert rc.json['code'] == data + + rc = client_get(client, f"{BASE_URI}/strategy/NoStrat") + assert_response(rc, 404) + + +def test_list_available_pairs(botclient): + ftbot, client = botclient + + rc = client_get(client, f"{BASE_URI}/available_pairs") + + assert_response(rc) + assert rc.json['length'] == 12 + assert isinstance(rc.json['pairs'], list) + + rc = client_get(client, f"{BASE_URI}/available_pairs?timeframe=5m") + assert_response(rc) + assert rc.json['length'] == 12 + + rc = client_get(client, f"{BASE_URI}/available_pairs?stake_currency=ETH") + assert_response(rc) + assert rc.json['length'] == 1 + assert rc.json['pairs'] == ['XRP/ETH'] + assert len(rc.json['pair_interval']) == 2 + + rc = client_get(client, f"{BASE_URI}/available_pairs?stake_currency=ETH&timeframe=5m") + assert_response(rc) + assert rc.json['length'] == 1 + assert rc.json['pairs'] == ['XRP/ETH'] + assert len(rc.json['pair_interval']) == 1 diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index edf6bae4d..06706120f 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -1,10 +1,10 @@ # pragma pylint: disable=missing-docstring, C0103 -import time import logging +import time from unittest.mock import MagicMock -from freqtrade.rpc import RPCMessageType, RPCManager -from tests.conftest import log_has, get_patched_freqtradebot +from freqtrade.rpc import RPCManager, RPCMessageType +from tests.conftest import get_patched_freqtradebot, log_has def test__init__(mocker, default_conf) -> None: @@ -124,10 +124,10 @@ def test_send_msg_webhook_CustomMessagetype(mocker, default_conf, caplog) -> Non rpc_manager = RPCManager(get_patched_freqtradebot(mocker, default_conf)) assert 'webhook' in [mod.name for mod in rpc_manager.registered_modules] - rpc_manager.send_msg({'type': RPCMessageType.CUSTOM_NOTIFICATION, + rpc_manager.send_msg({'type': RPCMessageType.STARTUP_NOTIFICATION, 'status': 'TestMessage'}) assert log_has( - "Message type RPCMessageType.CUSTOM_NOTIFICATION not implemented by handler webhook.", + "Message type 'startup' not implemented by handler webhook.", caplog) @@ -137,7 +137,7 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) - rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists, freqtradebot.protections) assert telegram_mock.call_count == 3 assert "*Exchange:* `bittrex`" in telegram_mock.call_args_list[1][0][0]['status'] @@ -147,10 +147,14 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: default_conf['whitelist'] = {'method': 'VolumePairList', 'config': {'number_assets': 20} } + default_conf['protections'] = [{"method": "StoplossGuard", + "lookback_period": 60, "trade_limit": 2, "stop_duration": 60}] + freqtradebot = get_patched_freqtradebot(mocker, default_conf) - rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) - assert telegram_mock.call_count == 3 + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists, freqtradebot.protections) + assert telegram_mock.call_count == 4 assert "Dry run is enabled." in telegram_mock.call_args_list[0][0][0]['status'] + assert 'StoplossGuard' in telegram_mock.call_args_list[-1][0][0]['status'] def test_init_apiserver_disabled(mocker, default_conf, caplog) -> None: diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index b18106ee5..5040f35cf 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -6,22 +6,25 @@ import re from datetime import datetime from random import choice, randint from string import ascii_uppercase -from unittest.mock import ANY, MagicMock, PropertyMock +from unittest.mock import ANY, MagicMock import arrow import pytest -from telegram import Chat, Message, Update +from telegram import Chat, Message, ReplyKeyboardMarkup, Update from telegram.error import NetworkError from freqtrade import __version__ +from freqtrade.constants import CANCEL_REASON from freqtrade.edge import PairInfo +from freqtrade.exceptions import OperationalException from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.persistence import Trade +from freqtrade.loggers import setup_logging +from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPCMessageType from freqtrade.rpc.telegram import Telegram, authorized_only -from freqtrade.state import State +from freqtrade.state import RunMode, State from freqtrade.strategy.interface import SellType -from tests.conftest import (get_patched_freqtradebot, log_has, patch_exchange, +from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, patch_exchange, patch_get_signal, patch_whitelist) @@ -51,20 +54,33 @@ class DummyCls(Telegram): raise Exception('test') -def test__init__(default_conf, mocker) -> None: +def get_telegram_testobject(mocker, default_conf, mock=True): + msg_mock = MagicMock() + if mock: + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + ftbot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(ftbot) + + return telegram, ftbot, msg_mock + + +def test_telegram__init__(default_conf, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) - assert telegram._updater is None + telegram, _, _ = get_telegram_testobject(mocker, default_conf) assert telegram._config == default_conf -def test_init(default_conf, mocker, caplog) -> None: +def test_telegram_init(default_conf, mocker, caplog) -> None: start_polling = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock(return_value=start_polling)) - Telegram(get_patched_freqtradebot(mocker, default_conf)) + get_telegram_testobject(mocker, default_conf, mock=False) assert start_polling.call_count == 0 # number of handles registered @@ -72,30 +88,28 @@ def test_init(default_conf, mocker, caplog) -> None: assert start_polling.start_polling.call_count == 1 message_str = ("rpc.telegram is listening for following commands: [['status'], ['profit'], " - "['balance'], ['start'], ['stop'], ['forcesell'], ['forcebuy'], " - "['performance'], ['daily'], ['count'], ['reload_conf'], ['show_config'], " - "['stopbuy'], ['whitelist'], ['blacklist'], ['edge'], ['help'], ['version']]") + "['balance'], ['start'], ['stop'], ['forcesell'], ['forcebuy'], ['trades'], " + "['delete'], ['performance'], ['stats'], ['daily'], ['count'], ['locks'], " + "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], ['stopbuy'], " + "['whitelist'], ['blacklist'], ['logs'], ['edge'], ['help'], ['version']" + "]") assert log_has(message_str, caplog) -def test_cleanup(default_conf, mocker) -> None: +def test_cleanup(default_conf, mocker, ) -> None: updater_mock = MagicMock() updater_mock.stop = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', updater_mock) - telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) + telegram, _, _ = get_telegram_testobject(mocker, default_conf, mock=False) telegram.cleanup() assert telegram._updater.stop.call_count == 1 -def test_authorized_only(default_conf, mocker, caplog) -> None: +def test_authorized_only(default_conf, mocker, caplog, update) -> None: patch_exchange(mocker) - chat = Chat(0, 0) - update = Update(randint(1, 100)) - update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) - default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) patch_get_signal(bot, (True, False)) @@ -111,7 +125,7 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: patch_exchange(mocker) chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) - update.message = Message(randint(1, 100), 0, datetime.utcnow(), chat) + update.message = Message(randint(1, 100), datetime.utcnow(), chat) default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) @@ -124,12 +138,9 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: assert not log_has('Exception occurred within Telegram module', caplog) -def test_authorized_only_exception(default_conf, mocker, caplog) -> None: +def test_authorized_only_exception(default_conf, mocker, caplog, update) -> None: patch_exchange(mocker) - update = Update(randint(1, 100)) - update.message = Message(randint(1, 100), 0, datetime.utcnow(), Chat(0, 0)) - default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) @@ -143,16 +154,14 @@ def test_authorized_only_exception(default_conf, mocker, caplog) -> None: assert log_has('Exception occurred within Telegram module', caplog) -def test_status(default_conf, update, mocker, fee, ticker,) -> None: +def test_telegram_status(default_conf, update, mocker) -> None: update.message.chat.id = "123" default_conf['telegram']['enabled'] = False default_conf['telegram']['chat_id'] = "123" - msg_mock = MagicMock() status_table = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), _rpc_trade_status=MagicMock(return_value=[{ 'trade_id': 1, 'pair': 'ETH/BTC', @@ -167,21 +176,22 @@ def test_status(default_conf, update, mocker, fee, ticker,) -> None: 'amount': 90.99181074, 'stake_amount': 90.99181074, 'close_profit_pct': None, - 'current_profit': -0.0059, - 'current_profit_pct': -0.59, - 'initial_stop_loss': 1.098e-05, - 'stop_loss': 1.099e-05, + 'profit': -0.0059, + 'profit_pct': -0.59, + 'initial_stop_loss_abs': 1.098e-05, + 'stop_loss_abs': 1.099e-05, 'sell_order_status': None, 'initial_stop_loss_pct': -0.05, + 'stoploss_current_dist': 1e-08, + 'stoploss_current_dist_pct': -0.02, 'stop_loss_pct': -0.01, - 'open_order': '(limit buy rem=0.00000000)' + 'open_order': '(limit buy rem=0.00000000)', + 'is_open': True }]), _status_table=status_table, - _send_msg=msg_mock ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._status(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -199,21 +209,16 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: fetch_ticker=ticker, get_fee=fee, ) - msg_mock = MagicMock() status_table = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), _status_table=status_table, - _send_msg=msg_mock ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - freqtradebot.state = State.STOPPED # Status is also enabled when stopped telegram._status(update=update, context=MagicMock()) @@ -247,21 +252,14 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) default_conf['stake_amount'] = 15.0 - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + + patch_get_signal(freqtradebot, (True, False)) freqtradebot.state = State.STOPPED # Status table is also enabled when stopped @@ -302,16 +300,10 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, fetch_ticker=ticker, get_fee=fee, ) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Create some test data freqtradebot.enter_positions() @@ -340,6 +332,18 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] + # Reset msg_mock + msg_mock.reset_mock() + context.args = [] + telegram._daily(update=update, context=context) + assert msg_mock.call_count == 1 + assert 'Daily' in msg_mock.call_args_list[0][0][0] + assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] + assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] + assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] + assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] + # Reset msg_mock msg_mock.reset_mock() freqtradebot.config['max_open_trades'] = 2 @@ -368,16 +372,9 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker ) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Try invalid data msg_mock.reset_mock() @@ -407,16 +404,9 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, fetch_ticker=ticker, get_fee=fee, ) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) telegram._profit(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -459,6 +449,33 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, assert '*Best Performing:* `ETH/BTC: 6.20%`' in msg_mock.call_args_list[-1][0][0] +def test_telegram_stats(default_conf, update, ticker, ticker_sell_up, fee, + limit_buy_order, limit_sell_order, mocker) -> None: + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_fee=fee, + ) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) + + telegram._stats(update=update, context=MagicMock()) + assert msg_mock.call_count == 1 + # assert 'No trades yet.' in msg_mock.call_args_list[0][0][0] + msg_mock.reset_mock() + + # Create some test data + create_mock_trades(fee) + + telegram._stats(update=update, context=MagicMock()) + assert msg_mock.call_count == 1 + assert 'Sell Reason' in msg_mock.call_args_list[-1][0][0] + assert 'ROI' in msg_mock.call_args_list[-1][0][0] + assert 'Avg. Duration' in msg_mock.call_args_list[-1][0][0] + msg_mock.reset_mock() + + def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tickers) -> None: default_conf['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance) @@ -466,18 +483,9 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination', side_effect=lambda a, b: f"{a}/{b}") - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 @@ -495,18 +503,9 @@ def test_balance_handle_empty_response(default_conf, update, mocker) -> None: default_conf['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={}) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - freqtradebot.config['dry_run'] = False telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] @@ -517,18 +516,9 @@ def test_balance_handle_empty_response(default_conf, update, mocker) -> None: def test_balance_handle_empty_response_dry(default_conf, update, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={}) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 @@ -555,18 +545,9 @@ def test_balance_handle_too_large_response(default_conf, update, mocker) -> None 'value': 1000.0, }) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - telegram._balance(update=update, context=MagicMock()) assert msg_mock.call_count > 1 # Test if wrap happens around 4000 - @@ -577,15 +558,8 @@ def test_balance_handle_too_large_response(default_conf, update, mocker) -> None def test_start_handle(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.STOPPED assert freqtradebot.state == State.STOPPED @@ -595,15 +569,8 @@ def test_start_handle(default_conf, update, mocker) -> None: def test_start_handle_already_running(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.RUNNING assert freqtradebot.state == State.RUNNING @@ -614,15 +581,8 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: def test_stop_handle(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.RUNNING assert freqtradebot.state == State.RUNNING @@ -633,15 +593,8 @@ def test_stop_handle(default_conf, update, mocker) -> None: def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.STOPPED assert freqtradebot.state == State.STOPPED @@ -652,47 +605,33 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: def test_stopbuy_handle(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) assert freqtradebot.config['max_open_trades'] != 0 telegram._stopbuy(update=update, context=MagicMock()) assert freqtradebot.config['max_open_trades'] == 0 assert msg_mock.call_count == 1 - assert 'No more buy will occur from now. Run /reload_conf to reset.' \ + assert 'No more buy will occur from now. Run /reload_config to reset.' \ in msg_mock.call_args_list[0][0][0] -def test_reload_conf_handle(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) +def test_reload_config_handle(default_conf, update, mocker) -> None: - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.RUNNING assert freqtradebot.state == State.RUNNING - telegram._reload_conf(update=update, context=MagicMock()) - assert freqtradebot.state == State.RELOAD_CONF + telegram._reload_config(update=update, context=MagicMock()) + assert freqtradebot.state == State.RELOAD_CONFIG assert msg_mock.call_count == 1 - assert 'reloading config' in msg_mock.call_args_list[0][0][0] + assert 'Reloading config' in msg_mock.call_args_list[0][0][0] -def test_forcesell_handle(default_conf, update, ticker, fee, - ticker_sell_up, mocker) -> None: +def test_telegram_forcesell_handle(default_conf, update, ticker, fee, + ticker_sell_up, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) - rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + msg_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_exchange(mocker) patch_whitelist(mocker, default_conf) @@ -720,15 +659,16 @@ def test_forcesell_handle(default_conf, update, ticker, fee, context.args = ["1"] telegram._forcesell(update=update, context=context) - assert rpc_mock.call_count == 2 - last_msg = rpc_mock.call_args_list[-1][0][0] + assert msg_mock.call_count == 3 + last_msg = msg_mock.call_args_list[-1][0][0] assert { 'type': RPCMessageType.SELL_NOTIFICATION, + 'trade_id': 1, 'exchange': 'Bittrex', 'pair': 'ETH/BTC', 'gain': 'profit', 'limit': 1.173e-05, - 'amount': 91.07468123861567, + 'amount': 91.07468123, 'order_type': 'limit', 'open_rate': 1.098e-05, 'current_rate': 1.173e-05, @@ -742,11 +682,11 @@ def test_forcesell_handle(default_conf, update, ticker, fee, } == last_msg -def test_forcesell_down_handle(default_conf, update, ticker, fee, - ticker_sell_down, mocker) -> None: +def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, + ticker_sell_down, mocker) -> None: mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + msg_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_exchange(mocker) patch_whitelist(mocker, default_conf) @@ -778,16 +718,17 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, context.args = ["1"] telegram._forcesell(update=update, context=context) - assert rpc_mock.call_count == 2 + assert msg_mock.call_count == 3 - last_msg = rpc_mock.call_args_list[-1][0][0] + last_msg = msg_mock.call_args_list[-1][0][0] assert { 'type': RPCMessageType.SELL_NOTIFICATION, + 'trade_id': 1, 'exchange': 'Bittrex', 'pair': 'ETH/BTC', 'gain': 'loss', 'limit': 1.043e-05, - 'amount': 91.07468123861567, + 'amount': 91.07468123, 'order_type': 'limit', 'open_rate': 1.098e-05, 'current_rate': 1.043e-05, @@ -805,7 +746,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None patch_exchange(mocker) mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + msg_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_whitelist(mocker, default_conf) mocker.patch.multiple( @@ -820,22 +761,24 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None # Create some test data freqtradebot.enter_positions() - rpc_mock.reset_mock() + msg_mock.reset_mock() # /forcesell all context = MagicMock() context.args = ["all"] telegram._forcesell(update=update, context=context) - assert rpc_mock.call_count == 4 - msg = rpc_mock.call_args_list[0][0][0] + # Called for each trade 3 times + assert msg_mock.call_count == 8 + msg = msg_mock.call_args_list[1][0][0] assert { 'type': RPCMessageType.SELL_NOTIFICATION, + 'trade_id': 1, 'exchange': 'Bittrex', 'pair': 'ETH/BTC', 'gain': 'loss', 'limit': 1.099e-05, - 'amount': 91.07468123861567, + 'amount': 91.07468123, 'order_type': 'limit', 'open_rate': 1.098e-05, 'current_rate': 1.099e-05, @@ -852,16 +795,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Trader is not running freqtradebot.state = State.STOPPED @@ -879,7 +815,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: context.args = [] telegram._forcesell(update=update, context=context) assert msg_mock.call_count == 1 - assert 'invalid argument' in msg_mock.call_args_list[0][0][0] + assert "You must specify a trade-id or 'all'." in msg_mock.call_args_list[0][0][0] # Invalid argument msg_mock.reset_mock() @@ -892,21 +828,14 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: assert 'invalid argument' in msg_mock.call_args_list[0][0][0] -def test_forcebuy_handle(default_conf, update, markets, mocker) -> None: +def test_forcebuy_handle(default_conf, update, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) - mocker.patch('freqtrade.rpc.telegram.Telegram._send_msg', MagicMock()) - mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - markets=PropertyMock(markets), - ) + fbuy_mock = MagicMock(return_value=None) mocker.patch('freqtrade.rpc.RPC._rpc_forcebuy', fbuy_mock) - freqtradebot = FreqtradeBot(default_conf) + telegram, freqtradebot, _ = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # /forcebuy ETH/BTC context = MagicMock() @@ -931,42 +860,29 @@ def test_forcebuy_handle(default_conf, update, markets, mocker) -> None: assert fbuy_mock.call_args_list[0][0][1] == 0.055 -def test_forcebuy_handle_exception(default_conf, update, markets, mocker) -> None: +def test_forcebuy_handle_exception(default_conf, update, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) - rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram._send_msg', MagicMock()) - mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - markets=PropertyMock(markets), - ) - freqtradebot = FreqtradeBot(default_conf) + + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) update.message.text = '/forcebuy ETH/Nonepair' telegram._forcebuy(update=update, context=MagicMock()) - assert rpc_mock.call_count == 1 - assert rpc_mock.call_args_list[0][0][0] == 'Forcebuy not enabled.' + assert msg_mock.call_count == 1 + assert msg_mock.call_args_list[0][0][0] == 'Forcebuy not enabled.' def test_performance_handle(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) + mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Create some test data freqtradebot.enter_positions() @@ -988,21 +904,13 @@ def test_performance_handle(default_conf, update, ticker, fee, def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, - buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED telegram._count(update=update, context=MagicMock()) @@ -1024,16 +932,32 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: assert msg in msg_mock.call_args_list[0][0][0] -def test_whitelist_static(default_conf, update, mocker) -> None: - msg_mock = MagicMock() +def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) + PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason') + PairLocks.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef') + + telegram._locks(update=update, context=MagicMock()) + + assert 'Pair' in msg_mock.call_args_list[0][0][0] + assert 'Until' in msg_mock.call_args_list[0][0][0] + assert 'Reason\n' in msg_mock.call_args_list[0][0][0] + assert 'ETH/BTC' in msg_mock.call_args_list[0][0][0] + assert 'XRP/BTC' in msg_mock.call_args_list[0][0][0] + assert 'deadbeef' in msg_mock.call_args_list[0][0][0] + assert 'randreason' in msg_mock.call_args_list[0][0][0] + + +def test_whitelist_static(default_conf, update, mocker) -> None: + + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1042,19 +966,11 @@ def test_whitelist_static(default_conf, update, mocker) -> None: def test_whitelist_dynamic(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 4 }] - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - - telegram = Telegram(freqtradebot) + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1063,15 +979,8 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: def test_blacklist_static(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._blacklist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1102,17 +1011,40 @@ def test_blacklist_static(default_conf, update, mocker) -> None: assert freqtradebot.pairlists.blacklist == ["DOGE/BTC", "HOT/BTC", "ETH/BTC"] -def test_edge_disabled(default_conf, update, mocker) -> None: - msg_mock = MagicMock() +def test_telegram_logs(default_conf, update, mocker) -> None: mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - _send_msg=msg_mock ) + setup_logging(default_conf) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) - telegram = Telegram(freqtradebot) + context = MagicMock() + context.args = [] + telegram._logs(update=update, context=context) + assert msg_mock.call_count == 1 + assert "freqtrade\\.rpc\\.telegram" in msg_mock.call_args_list[0][0][0] + + msg_mock.reset_mock() + context.args = ["1"] + telegram._logs(update=update, context=context) + assert msg_mock.call_count == 1 + + msg_mock.reset_mock() + # Test with changed MaxMessageLength + mocker.patch('freqtrade.rpc.telegram.MAX_TELEGRAM_MESSAGE_LENGTH', 200) + context = MagicMock() + context.args = [] + telegram._logs(update=update, context=context) + # Called at least 2 times. Exact times will change with unrelated changes to setup messages + # Therefore we don't test for this explicitly. + assert msg_mock.call_count >= 2 + + +def test_edge_disabled(default_conf, update, mocker) -> None: + + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._edge(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1120,21 +1052,13 @@ def test_edge_disabled(default_conf, update, mocker) -> None: def test_edge_enabled(edge_conf, update, mocker) -> None: - msg_mock = MagicMock() mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), } )) - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, edge_conf) - - telegram = Telegram(freqtradebot) + telegram, _, msg_mock = get_telegram_testobject(mocker, edge_conf) telegram._edge(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1142,16 +1066,58 @@ def test_edge_enabled(edge_conf, update, mocker) -> None: assert 'Pair Winrate Expectancy Stoploss' in msg_mock.call_args_list[0][0][0] -def test_help_handle(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) +def test_telegram_trades(mocker, update, default_conf, fee): - telegram = Telegram(freqtradebot) + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + + context = MagicMock() + context.args = [] + + telegram._trades(update=update, context=context) + assert "0 recent trades:" in msg_mock.call_args_list[0][0][0] + assert "
" not in msg_mock.call_args_list[0][0][0]
+    msg_mock.reset_mock()
+
+    context.args = ['hello']
+    telegram._trades(update=update, context=context)
+    assert "0 recent trades:" in msg_mock.call_args_list[0][0][0]
+    assert "
" not in msg_mock.call_args_list[0][0][0]
+    msg_mock.reset_mock()
+
+    create_mock_trades(fee)
+
+    context = MagicMock()
+    context.args = [5]
+    telegram._trades(update=update, context=context)
+    msg_mock.call_count == 1
+    assert "2 recent trades:" in msg_mock.call_args_list[0][0][0]
+    assert "Profit (" in msg_mock.call_args_list[0][0][0]
+    assert "Open Date" in msg_mock.call_args_list[0][0][0]
+    assert "
" in msg_mock.call_args_list[0][0][0]
+
+
+def test_telegram_delete_trade(mocker, update, default_conf, fee):
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+    context = MagicMock()
+    context.args = []
+
+    telegram._delete_trade(update=update, context=context)
+    assert "Trade-id not set." in msg_mock.call_args_list[0][0][0]
+
+    msg_mock.reset_mock()
+    create_mock_trades(fee)
+
+    context = MagicMock()
+    context.args = [1]
+    telegram._delete_trade(update=update, context=context)
+    msg_mock.call_count == 1
+    assert "Deleted trade 1." in msg_mock.call_args_list[0][0][0]
+    assert "Please make sure to take care of this asset" in msg_mock.call_args_list[0][0][0]
+
+
+def test_help_handle(default_conf, update, mocker) -> None:
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram._help(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
@@ -1159,14 +1125,8 @@ def test_help_handle(default_conf, update, mocker) -> None:
 
 
 def test_version_handle(default_conf, update, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram._version(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
@@ -1174,14 +1134,10 @@ def test_version_handle(default_conf, update, mocker) -> None:
 
 
 def test_show_config_handle(default_conf, update, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    default_conf['runmode'] = RunMode.DRY_RUN
+
+    telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram._show_config(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
@@ -1200,16 +1156,9 @@ def test_show_config_handle(default_conf, update, mocker) -> None:
     assert '*Initial Stoploss:* `-0.1`' in msg_mock.call_args_list[0][0][0]
 
 
-def test_send_msg_buy_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
-    telegram.send_msg({
+def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None:
+
+    msg = {
         'type': RPCMessageType.BUY_NOTIFICATION,
         'exchange': 'Bittrex',
         'pair': 'ETH/BTC',
@@ -1222,7 +1171,10 @@ def test_send_msg_buy_notification(default_conf, mocker) -> None:
         'current_rate': 1.099e-05,
         'amount': 1333.3333333333335,
         'open_date': arrow.utcnow().shift(hours=-1)
-    })
+    }
+    telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf)
+
+    telegram.send_msg(msg)
     assert msg_mock.call_args[0][0] \
         == '\N{LARGE BLUE CIRCLE} *Bittrex:* Buying ETH/BTC\n' \
            '*Amount:* `1333.33333333`\n' \
@@ -1230,34 +1182,40 @@ def test_send_msg_buy_notification(default_conf, mocker) -> None:
            '*Current Rate:* `0.00001099`\n' \
            '*Total:* `(0.001000 BTC, 12.345 USD)`'
 
+    freqtradebot.config['telegram']['notification_settings'] = {'buy': 'off'}
+    caplog.clear()
+    msg_mock.reset_mock()
+    telegram.send_msg(msg)
+    msg_mock.call_count == 0
+    log_has("Notification 'buy' not sent.", caplog)
+
+    freqtradebot.config['telegram']['notification_settings'] = {'buy': 'silent'}
+    caplog.clear()
+    msg_mock.reset_mock()
+
+    telegram.send_msg(msg)
+    msg_mock.call_count == 1
+    msg_mock.call_args_list[0][1]['disable_notification'] is True
+
 
 def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
     telegram.send_msg({
         'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
         'exchange': 'Bittrex',
         'pair': 'ETH/BTC',
+        'reason': CANCEL_REASON['TIMEOUT']
     })
-    assert msg_mock.call_args[0][0] \
-        == ('\N{WARNING SIGN} *Bittrex:* Cancelling Open Buy Order for ETH/BTC')
+    assert (msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Bittrex:* '
+            'Cancelling open buy Order for ETH/BTC. Reason: cancelled due to timeout.')
 
 
 def test_send_msg_sell_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
     old_convamount = telegram._fiat_converter.convert_amount
     telegram._fiat_converter.convert_amount = lambda a, b, c: -24.812
     telegram.send_msg({
@@ -1320,14 +1278,9 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None:
 
 
 def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
     old_convamount = telegram._fiat_converter.convert_amount
     telegram._fiat_converter.convert_amount = lambda a, b, c: -24.812
     telegram.send_msg({
@@ -1354,14 +1307,8 @@ def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None:
 
 
 def test_send_msg_status_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     telegram.send_msg({
         'type': RPCMessageType.STATUS_NOTIFICATION,
         'status': 'running'
@@ -1370,14 +1317,7 @@ def test_send_msg_status_notification(default_conf, mocker) -> None:
 
 
 def test_warning_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     telegram.send_msg({
         'type': RPCMessageType.WARNING_NOTIFICATION,
         'status': 'message'
@@ -1385,31 +1325,17 @@ def test_warning_notification(default_conf, mocker) -> None:
     assert msg_mock.call_args[0][0] == '\N{WARNING SIGN} *Warning:* `message`'
 
 
-def test_custom_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+def test_startup_notification(default_conf, mocker) -> None:
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     telegram.send_msg({
-        'type': RPCMessageType.CUSTOM_NOTIFICATION,
+        'type': RPCMessageType.STARTUP_NOTIFICATION,
         'status': '*Custom:* `Hello World`'
     })
     assert msg_mock.call_args[0][0] == '*Custom:* `Hello World`'
 
 
 def test_send_msg_unknown_type(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, _ = get_telegram_testobject(mocker, default_conf)
     with pytest.raises(NotImplementedError, match=r'Unknown message type: None'):
         telegram.send_msg({
             'type': None,
@@ -1418,14 +1344,8 @@ def test_send_msg_unknown_type(default_conf, mocker) -> None:
 
 def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None:
     del default_conf['fiat_display_currency']
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
     telegram.send_msg({
         'type': RPCMessageType.BUY_NOTIFICATION,
         'exchange': 'Bittrex',
@@ -1449,14 +1369,8 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None:
 
 def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None:
     del default_conf['fiat_display_currency']
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
     telegram.send_msg({
         'type': RPCMessageType.SELL_NOTIFICATION,
         'exchange': 'Binance',
@@ -1496,14 +1410,8 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None:
 ])
 def test__sell_emoji(default_conf, mocker, msg, expected):
     del default_conf['fiat_display_currency']
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, _ = get_telegram_testobject(mocker, default_conf)
 
     assert telegram._get_sell_emoji(msg) == expected
 
@@ -1511,8 +1419,7 @@ def test__sell_emoji(default_conf, mocker, msg, expected):
 def test__send_msg(default_conf, mocker) -> None:
     mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock())
     bot = MagicMock()
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, _ = get_telegram_testobject(mocker, default_conf, mock=False)
     telegram._updater = MagicMock()
     telegram._updater.bot = bot
 
@@ -1525,8 +1432,7 @@ def test__send_msg_network_error(default_conf, mocker, caplog) -> None:
     mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock())
     bot = MagicMock()
     bot.send_message = MagicMock(side_effect=NetworkError('Oh snap'))
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, _ = get_telegram_testobject(mocker, default_conf, mock=False)
     telegram._updater = MagicMock()
     telegram._updater.bot = bot
 
@@ -1536,3 +1442,53 @@ def test__send_msg_network_error(default_conf, mocker, caplog) -> None:
     # Bot should've tried to send it twice
     assert len(bot.method_calls) == 2
     assert log_has('Telegram NetworkError: Oh snap! Trying one more time.', caplog)
+
+
+def test__send_msg_keyboard(default_conf, mocker, caplog) -> None:
+    mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock())
+    bot = MagicMock()
+    bot.send_message = MagicMock()
+    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
+
+    invalid_keys_list = [['/not_valid', '/profit'], ['/daily'], ['/alsoinvalid']]
+    default_keys_list = [['/daily', '/profit', '/balance'],
+                         ['/status', '/status table', '/performance'],
+                         ['/count', '/start', '/stop', '/help']]
+    default_keyboard = ReplyKeyboardMarkup(default_keys_list)
+
+    custom_keys_list = [['/daily', '/stats', '/balance', '/profit'],
+                        ['/count', '/start', '/reload_config', '/help']]
+    custom_keyboard = ReplyKeyboardMarkup(custom_keys_list)
+
+    def init_telegram(freqtradebot):
+        telegram = Telegram(freqtradebot)
+        telegram._updater = MagicMock()
+        telegram._updater.bot = bot
+        return telegram
+
+    # no keyboard in config -> default keyboard
+    freqtradebot.config['telegram']['enabled'] = True
+    telegram = init_telegram(freqtradebot)
+    telegram._send_msg('test')
+    used_keyboard = bot.send_message.call_args[1]['reply_markup']
+    assert used_keyboard == default_keyboard
+
+    # invalid keyboard in config -> default keyboard
+    freqtradebot.config['telegram']['enabled'] = True
+    freqtradebot.config['telegram']['keyboard'] = invalid_keys_list
+    err_msg = re.escape("config.telegram.keyboard: Invalid commands for custom "
+                        "Telegram keyboard: ['/not_valid', '/alsoinvalid']"
+                        "\nvalid commands are: ") + r"*"
+    with pytest.raises(OperationalException, match=err_msg):
+        telegram = init_telegram(freqtradebot)
+
+    # valid keyboard in config -> custom keyboard
+    freqtradebot.config['telegram']['enabled'] = True
+    freqtradebot.config['telegram']['keyboard'] = custom_keys_list
+    telegram = init_telegram(freqtradebot)
+    telegram._send_msg('test')
+    used_keyboard = bot.send_message.call_args[1]['reply_markup']
+    assert used_keyboard == custom_keyboard
+    assert log_has("using custom keyboard from config.json: "
+                   "[['/daily', '/stats', '/balance', '/profit'], ['/count', "
+                   "'/start', '/reload_config', '/help']]", caplog)
diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py
index 1ced62746..9256a5316 100644
--- a/tests/rpc/test_rpc_webhook.py
+++ b/tests/rpc/test_rpc_webhook.py
@@ -150,7 +150,7 @@ def test_send_msg(default_conf, mocker):
             default_conf["webhook"]["webhooksellcancel"]["value3"].format(**msg))
     for msgtype in [RPCMessageType.STATUS_NOTIFICATION,
                     RPCMessageType.WARNING_NOTIFICATION,
-                    RPCMessageType.CUSTOM_NOTIFICATION]:
+                    RPCMessageType.STARTUP_NOTIFICATION]:
         # Test notification
         msg = {
             'type': msgtype,
@@ -174,7 +174,7 @@ def test_exception_send_msg(default_conf, mocker, caplog):
 
     webhook = Webhook(get_patched_freqtradebot(mocker, default_conf))
     webhook.send_msg({'type': RPCMessageType.BUY_NOTIFICATION})
-    assert log_has(f"Message type {RPCMessageType.BUY_NOTIFICATION} not configured for webhooks",
+    assert log_has(f"Message type '{RPCMessageType.BUY_NOTIFICATION}' not configured for webhooks",
                    caplog)
 
     default_conf["webhook"] = get_webhook_dict()
diff --git a/tests/strategy/strats/default_strategy.py b/tests/strategy/strats/default_strategy.py
index 7ea55d3f9..98842ff7c 100644
--- a/tests/strategy/strats/default_strategy.py
+++ b/tests/strategy/strats/default_strategy.py
@@ -29,7 +29,7 @@ class DefaultStrategy(IStrategy):
     stoploss = -0.10
 
     # Optimal ticker interval for the strategy
-    ticker_interval = '5m'
+    timeframe = '5m'
 
     # Optional order type mapping
     order_types = {
diff --git a/tests/strategy/strats/legacy_strategy.py b/tests/strategy/strats/legacy_strategy.py
index 89ce3f8cb..1e7bb5e1e 100644
--- a/tests/strategy/strats/legacy_strategy.py
+++ b/tests/strategy/strats/legacy_strategy.py
@@ -1,12 +1,13 @@
 
 # --- Do not remove these libs ---
-from freqtrade.strategy.interface import IStrategy
-from pandas import DataFrame
-# --------------------------------
-
 # Add your lib to import here
 import talib.abstract as ta
+from pandas import DataFrame
 
+from freqtrade.strategy.interface import IStrategy
+
+
+# --------------------------------
 
 # This class is a sample. Feel free to customize it.
 class TestStrategyLegacy(IStrategy):
@@ -31,6 +32,7 @@ class TestStrategyLegacy(IStrategy):
     stoploss = -0.10
 
     # Optimal ticker interval for the strategy
+    # Keep the legacy value here to test compatibility
     ticker_interval = '5m'
 
     def populate_indicators(self, dataframe: DataFrame) -> DataFrame:
diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py
index 0b8ea9f85..1b1648db9 100644
--- a/tests/strategy/test_default_strategy.py
+++ b/tests/strategy/test_default_strategy.py
@@ -6,7 +6,7 @@ from .strats.default_strategy import DefaultStrategy
 def test_default_strategy_structure():
     assert hasattr(DefaultStrategy, 'minimal_roi')
     assert hasattr(DefaultStrategy, 'stoploss')
-    assert hasattr(DefaultStrategy, 'ticker_interval')
+    assert hasattr(DefaultStrategy, 'timeframe')
     assert hasattr(DefaultStrategy, 'populate_indicators')
     assert hasattr(DefaultStrategy, 'populate_buy_trend')
     assert hasattr(DefaultStrategy, 'populate_sell_trend')
@@ -18,7 +18,7 @@ def test_default_strategy(result):
     metadata = {'pair': 'ETH/BTC'}
     assert type(strategy.minimal_roi) is dict
     assert type(strategy.stoploss) is float
-    assert type(strategy.ticker_interval) is str
+    assert type(strategy.timeframe) is str
     indicators = strategy.populate_indicators(result, metadata)
     assert type(indicators) is DataFrame
     assert type(strategy.populate_buy_trend(indicators, metadata)) is DataFrame
diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py
index e5539099b..640849ba4 100644
--- a/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@@ -1,6 +1,6 @@
 # pragma pylint: disable=missing-docstring, C0103
-
 import logging
+from datetime import datetime, timedelta, timezone
 from unittest.mock import MagicMock
 
 import arrow
@@ -8,17 +8,20 @@ import pytest
 from pandas import DataFrame
 
 from freqtrade.configuration import TimeRange
+from freqtrade.data.dataprovider import DataProvider
 from freqtrade.data.history import load_data
 from freqtrade.exceptions import StrategyError
-from freqtrade.persistence import Trade
+from freqtrade.persistence import PairLocks, Trade
 from freqtrade.resolvers import StrategyResolver
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
-from tests.conftest import get_patched_exchange, log_has, log_has_re
+from tests.conftest import log_has, log_has_re
 
 from .strats.default_strategy import DefaultStrategy
 
+
 # Avoid to reinit the same object again and again
 _STRATEGY = DefaultStrategy(config={})
+_STRATEGY.dp = DataProvider({}, None, None)
 
 
 def test_returns_latest_signal(mocker, default_conf, ohlcv_history):
@@ -29,63 +32,60 @@ def test_returns_latest_signal(mocker, default_conf, ohlcv_history):
     mocked_history['buy'] = 0
     mocked_history.loc[1, 'sell'] = 1
 
-    mocker.patch.object(
-        _STRATEGY, '_analyze_ticker_internal',
-        return_value=mocked_history
-    )
-
-    assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (False, True)
+    assert _STRATEGY.get_signal('ETH/BTC', '5m', mocked_history) == (False, True)
     mocked_history.loc[1, 'sell'] = 0
     mocked_history.loc[1, 'buy'] = 1
 
-    mocker.patch.object(
-        _STRATEGY, '_analyze_ticker_internal',
-        return_value=mocked_history
-    )
-    assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (True, False)
+    assert _STRATEGY.get_signal('ETH/BTC', '5m', mocked_history) == (True, False)
     mocked_history.loc[1, 'sell'] = 0
     mocked_history.loc[1, 'buy'] = 0
 
-    mocker.patch.object(
-        _STRATEGY, '_analyze_ticker_internal',
-        return_value=mocked_history
-    )
-    assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (False, False)
+    assert _STRATEGY.get_signal('ETH/BTC', '5m', mocked_history) == (False, False)
 
 
-def test_get_signal_empty(default_conf, mocker, caplog):
-    assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'],
-                                                  DataFrame())
-    assert log_has('Empty candle (OHLCV) data for pair foo', caplog)
-    caplog.clear()
-
-    assert (False, False) == _STRATEGY.get_signal('bar', default_conf['ticker_interval'],
-                                                  [])
-    assert log_has('Empty candle (OHLCV) data for pair bar', caplog)
-
-
-def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ohlcv_history):
-    caplog.set_level(logging.INFO)
-    mocker.patch.object(
-        _STRATEGY, '_analyze_ticker_internal',
-        side_effect=ValueError('xyz')
-    )
-    assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'],
-                                                  ohlcv_history)
-    assert log_has_re(r'Strategy caused the following exception: xyz.*', caplog)
-
-
-def test_get_signal_empty_dataframe(default_conf, mocker, caplog, ohlcv_history):
-    caplog.set_level(logging.INFO)
+def test_analyze_pair_empty(default_conf, mocker, caplog, ohlcv_history):
+    mocker.patch.object(_STRATEGY.dp, 'ohlcv', return_value=ohlcv_history)
     mocker.patch.object(
         _STRATEGY, '_analyze_ticker_internal',
         return_value=DataFrame([])
     )
     mocker.patch.object(_STRATEGY, 'assert_df')
 
-    assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
-                                                  ohlcv_history)
-    assert log_has('Empty dataframe for pair xyz', caplog)
+    _STRATEGY.analyze_pair('ETH/BTC')
+
+    assert log_has('Empty dataframe for pair ETH/BTC', caplog)
+
+
+def test_get_signal_empty(default_conf, mocker, caplog):
+    assert (False, False) == _STRATEGY.get_signal('foo', default_conf['timeframe'], DataFrame())
+    assert log_has('Empty candle (OHLCV) data for pair foo', caplog)
+    caplog.clear()
+
+    assert (False, False) == _STRATEGY.get_signal('bar', default_conf['timeframe'], None)
+    assert log_has('Empty candle (OHLCV) data for pair bar', caplog)
+    caplog.clear()
+
+    assert (False, False) == _STRATEGY.get_signal('baz', default_conf['timeframe'], DataFrame([]))
+    assert log_has('Empty candle (OHLCV) data for pair baz', caplog)
+
+
+def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ohlcv_history):
+    caplog.set_level(logging.INFO)
+    mocker.patch.object(_STRATEGY.dp, 'ohlcv', return_value=ohlcv_history)
+    mocker.patch.object(
+        _STRATEGY, '_analyze_ticker_internal',
+        side_effect=ValueError('xyz')
+    )
+    _STRATEGY.analyze_pair('foo')
+    assert log_has_re(r'Strategy caused the following exception: xyz.*', caplog)
+    caplog.clear()
+
+    mocker.patch.object(
+        _STRATEGY, 'analyze_ticker',
+        side_effect=Exception('invalid ticker history ')
+    )
+    _STRATEGY.analyze_pair('foo')
+    assert log_has_re(r'Strategy caused the following exception: xyz.*', caplog)
 
 
 def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history):
@@ -99,13 +99,9 @@ def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history):
     mocked_history.loc[1, 'buy'] = 1
 
     caplog.set_level(logging.INFO)
-    mocker.patch.object(
-        _STRATEGY, '_analyze_ticker_internal',
-        return_value=mocked_history
-    )
     mocker.patch.object(_STRATEGY, 'assert_df')
-    assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
-                                                  ohlcv_history)
+
+    assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['timeframe'], mocked_history)
     assert log_has('Outdated history for pair xyz. Last tick is 16 minutes old', caplog)
 
 
@@ -120,52 +116,46 @@ def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history):
     mocked_history.loc[1, 'buy'] = 1
 
     caplog.set_level(logging.INFO)
+    mocker.patch.object(_STRATEGY.dp, 'ohlcv', return_value=ohlcv_history)
+    mocker.patch.object(_STRATEGY.dp, 'get_analyzed_dataframe', return_value=(mocked_history, 0))
     mocker.patch.object(
         _STRATEGY, 'assert_df',
         side_effect=StrategyError('Dataframe returned...')
     )
-    assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'],
-                                                  ohlcv_history)
+    _STRATEGY.analyze_pair('xyz')
     assert log_has('Unable to analyze candle (OHLCV) data for pair xyz: Dataframe returned...',
                    caplog)
 
 
 def test_assert_df(default_conf, mocker, ohlcv_history, caplog):
+    df_len = len(ohlcv_history) - 1
     # Ensure it's running when passed correctly
     _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                        ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date'])
+                        ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[df_len, 'date'])
 
     with pytest.raises(StrategyError, match=r"Dataframe returned from strategy.*length\."):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history) + 1,
-                            ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date'])
+                            ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[df_len, 'date'])
 
     with pytest.raises(StrategyError,
                        match=r"Dataframe returned from strategy.*last close price\."):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                            ohlcv_history.loc[1, 'close'] + 0.01, ohlcv_history.loc[1, 'date'])
+                            ohlcv_history.loc[df_len, 'close'] + 0.01,
+                            ohlcv_history.loc[df_len, 'date'])
     with pytest.raises(StrategyError,
                        match=r"Dataframe returned from strategy.*last date\."):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                            ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date'])
+                            ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date'])
 
     _STRATEGY.disable_dataframe_checks = True
     caplog.clear()
     _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                        ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date'])
+                        ohlcv_history.loc[2, 'close'], ohlcv_history.loc[0, 'date'])
     assert log_has_re(r"Dataframe returned from strategy.*last date\.", caplog)
     # reset to avoid problems in other tests due to test leakage
     _STRATEGY.disable_dataframe_checks = False
 
 
-def test_get_signal_handles_exceptions(mocker, default_conf):
-    exchange = get_patched_exchange(mocker, default_conf)
-    mocker.patch.object(
-        _STRATEGY, 'analyze_ticker',
-        side_effect=Exception('invalid ticker history ')
-    )
-    assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, False)
-
-
 def test_ohlcvdata_to_dataframe(default_conf, testdatadir) -> None:
     default_conf.update({'strategy': 'DefaultStrategy'})
     strategy = StrategyResolver.load_strategy(default_conf)
@@ -274,14 +264,14 @@ def test_min_roi_reached3(default_conf, fee) -> None:
     strategy = StrategyResolver.load_strategy(default_conf)
     strategy.minimal_roi = min_roi
     trade = Trade(
-            pair='ETH/BTC',
-            stake_amount=0.001,
-            amount=5,
-            open_date=arrow.utcnow().shift(hours=-1).datetime,
-            fee_open=fee.return_value,
-            fee_close=fee.return_value,
-            exchange='bittrex',
-            open_rate=1,
+        pair='ETH/BTC',
+        stake_amount=0.001,
+        amount=5,
+        open_date=arrow.utcnow().shift(hours=-1).datetime,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        exchange='bittrex',
+        open_rate=1,
     )
 
     assert not strategy.min_roi_reached(trade, 0.02, arrow.utcnow().shift(minutes=-56).datetime)
@@ -342,6 +332,7 @@ def test__analyze_ticker_internal_skip_analyze(ohlcv_history, mocker, caplog) ->
 
     )
     strategy = DefaultStrategy({})
+    strategy.dp = DataProvider({}, None, None)
     strategy.process_only_new_candles = True
 
     ret = strategy._analyze_ticker_internal(ohlcv_history, {'pair': 'ETH/BTC'})
@@ -370,23 +361,20 @@ def test__analyze_ticker_internal_skip_analyze(ohlcv_history, mocker, caplog) ->
     assert log_has('Skipping TA Analysis for already analyzed candle', caplog)
 
 
+@pytest.mark.usefixtures("init_persistence")
 def test_is_pair_locked(default_conf):
     default_conf.update({'strategy': 'DefaultStrategy'})
+    PairLocks.timeframe = default_conf['timeframe']
     strategy = StrategyResolver.load_strategy(default_conf)
-    # dict should be empty
-    assert not strategy._pair_locked_until
+    # No lock should be present
+    assert len(PairLocks.get_pair_locks(None)) == 0
 
     pair = 'ETH/BTC'
     assert not strategy.is_pair_locked(pair)
-    strategy.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
+    strategy.lock_pair(pair, arrow.now(timezone.utc).shift(minutes=4).datetime)
     # ETH/BTC locked for 4 minutes
     assert strategy.is_pair_locked(pair)
 
-    # Test lock does not change
-    lock = strategy._pair_locked_until[pair]
-    strategy.lock_pair(pair, arrow.utcnow().shift(minutes=2).datetime)
-    assert lock == strategy._pair_locked_until[pair]
-
     # XRP/BTC should not be locked now
     pair = 'XRP/BTC'
     assert not strategy.is_pair_locked(pair)
@@ -399,6 +387,40 @@ def test_is_pair_locked(default_conf):
     strategy.unlock_pair(pair)
     assert not strategy.is_pair_locked(pair)
 
+    pair = 'BTC/USDT'
+    # Lock until 14:30
+    lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc)
+    # Subtract 2 seconds, as locking rounds up to the next candle.
+    strategy.lock_pair(pair, lock_time - timedelta(seconds=2))
+
+    assert not strategy.is_pair_locked(pair)
+    # latest candle is from 14:20, lock goes to 14:30
+    assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-10))
+    assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
+
+    # latest candle is from 14:25 (lock should be lifted)
+    # Since this is the "new candle" available at 14:30
+    assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-4))
+
+    # Should not be locked after time expired
+    assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=10))
+
+    # Change timeframe to 15m
+    strategy.timeframe = '15m'
+    # Candle from 14:14 - lock goes until 14:30
+    assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-16))
+    assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-15, seconds=-2))
+    # Candle from 14:15 - lock goes until 14:30
+    assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-15))
+
+
+def test_is_informative_pairs_callback(default_conf):
+    default_conf.update({'strategy': 'TestStrategyLegacy'})
+    strategy = StrategyResolver.load_strategy(default_conf)
+    # Should return empty
+    # Uses fallback to base implementation
+    assert [] == strategy.informative_pairs()
+
 
 @pytest.mark.parametrize('error', [
     ValueError, KeyError, Exception,
@@ -419,6 +441,11 @@ def test_strategy_safe_wrapper_error(caplog, error):
     assert isinstance(ret, bool)
     assert ret
 
+    caplog.clear()
+    # Test supressing error
+    ret = strategy_safe_wrapper(failing_method, message='DeadBeef', supress_error=True)()
+    assert log_has_re(r'DeadBeef.*', caplog)
+
 
 @pytest.mark.parametrize('value', [
     1, 22, 55, True, False, {'a': 1, 'b': '112'},
diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py
index 13ca68bf0..1c692d2da 100644
--- a/tests/strategy/test_strategy.py
+++ b/tests/strategy/test_strategy.py
@@ -18,13 +18,15 @@ def test_search_strategy():
 
     s, _ = StrategyResolver._search_object(
         directory=default_location,
-        object_name='DefaultStrategy'
+        object_name='DefaultStrategy',
+        add_source=True,
     )
     assert issubclass(s, IStrategy)
 
     s, _ = StrategyResolver._search_object(
         directory=default_location,
-        object_name='NotFoundStrategy'
+        object_name='NotFoundStrategy',
+        add_source=True,
     )
     assert s is None
 
@@ -53,6 +55,9 @@ def test_load_strategy(default_conf, result):
                          'strategy_path': str(Path(__file__).parents[2] / 'freqtrade/templates')
                          })
     strategy = StrategyResolver.load_strategy(default_conf)
+    assert isinstance(strategy.__source__, str)
+    assert 'class SampleStrategy' in strategy.__source__
+    assert isinstance(strategy.__file__, str)
     assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'})
 
 
@@ -105,8 +110,9 @@ def test_strategy(result, default_conf):
     assert strategy.stoploss == -0.10
     assert default_conf['stoploss'] == -0.10
 
+    assert strategy.timeframe == '5m'
     assert strategy.ticker_interval == '5m'
-    assert default_conf['ticker_interval'] == '5m'
+    assert default_conf['timeframe'] == '5m'
 
     df_indicators = strategy.advise_indicators(result, metadata=metadata)
     assert 'adx' in df_indicators
@@ -176,19 +182,19 @@ def test_strategy_override_trailing_stop_positive(caplog, default_conf):
                    caplog)
 
 
-def test_strategy_override_ticker_interval(caplog, default_conf):
+def test_strategy_override_timeframe(caplog, default_conf):
     caplog.set_level(logging.INFO)
 
     default_conf.update({
         'strategy': 'DefaultStrategy',
-        'ticker_interval': 60,
+        'timeframe': 60,
         'stake_currency': 'ETH'
     })
     strategy = StrategyResolver.load_strategy(default_conf)
 
-    assert strategy.ticker_interval == 60
+    assert strategy.timeframe == 60
     assert strategy.stake_currency == 'ETH'
-    assert log_has("Override strategy 'ticker_interval' with value in config file: 60.",
+    assert log_has("Override strategy 'timeframe' with value in config file: 60.",
                    caplog)
 
 
@@ -357,8 +363,9 @@ def test_deprecate_populate_indicators(result, default_conf):
 
 
 @pytest.mark.filterwarnings("ignore:deprecated")
-def test_call_deprecated_function(result, monkeypatch, default_conf):
+def test_call_deprecated_function(result, monkeypatch, default_conf, caplog):
     default_location = Path(__file__).parent / "strats"
+    del default_conf['timeframe']
     default_conf.update({'strategy': 'TestStrategyLegacy',
                          'strategy_path': default_location})
     strategy = StrategyResolver.load_strategy(default_conf)
@@ -369,6 +376,8 @@ def test_call_deprecated_function(result, monkeypatch, default_conf):
     assert strategy._buy_fun_len == 2
     assert strategy._sell_fun_len == 2
     assert strategy.INTERFACE_VERSION == 1
+    assert strategy.timeframe == '5m'
+    assert strategy.ticker_interval == '5m'
 
     indicator_df = strategy.advise_indicators(result, metadata=metadata)
     assert isinstance(indicator_df, DataFrame)
@@ -382,6 +391,9 @@ def test_call_deprecated_function(result, monkeypatch, default_conf):
     assert isinstance(selldf, DataFrame)
     assert 'sell' in selldf
 
+    assert log_has("DEPRECATED: Please migrate to using 'timeframe' instead of 'ticker_interval'.",
+                   caplog)
+
 
 def test_strategy_interface_versioning(result, monkeypatch, default_conf):
     default_conf.update({'strategy': 'DefaultStrategy'})
diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py
new file mode 100644
index 000000000..1d3e80d24
--- /dev/null
+++ b/tests/strategy/test_strategy_helpers.py
@@ -0,0 +1,88 @@
+import numpy as np
+import pandas as pd
+
+from freqtrade.strategy import merge_informative_pair, timeframe_to_minutes
+
+
+def generate_test_data(timeframe: str, size: int):
+    np.random.seed(42)
+    tf_mins = timeframe_to_minutes(timeframe)
+
+    base = np.random.normal(20, 2, size=size)
+
+    date = pd.period_range('2020-07-05', periods=size, freq=f'{tf_mins}min').to_timestamp()
+    df = pd.DataFrame({
+        'date': date,
+        'open': base,
+        'high': base + np.random.normal(2, 1, size=size),
+        'low': base - np.random.normal(2, 1, size=size),
+        'close': base + np.random.normal(0, 1, size=size),
+        'volume': np.random.normal(200, size=size)
+    }
+    )
+    df = df.dropna()
+    return df
+
+
+def test_merge_informative_pair():
+    data = generate_test_data('15m', 40)
+    informative = generate_test_data('1h', 40)
+
+    result = merge_informative_pair(data, informative, '15m', '1h', ffill=True)
+    assert isinstance(result, pd.DataFrame)
+    assert len(result) == len(data)
+    assert 'date' in result.columns
+    assert result['date'].equals(data['date'])
+    assert 'date_1h' in result.columns
+
+    assert 'open' in result.columns
+    assert 'open_1h' in result.columns
+    assert result['open'].equals(data['open'])
+
+    assert 'close' in result.columns
+    assert 'close_1h' in result.columns
+    assert result['close'].equals(data['close'])
+
+    assert 'volume' in result.columns
+    assert 'volume_1h' in result.columns
+    assert result['volume'].equals(data['volume'])
+
+    # First 4 rows are empty
+    assert result.iloc[0]['date_1h'] is pd.NaT
+    assert result.iloc[1]['date_1h'] is pd.NaT
+    assert result.iloc[2]['date_1h'] is pd.NaT
+    assert result.iloc[3]['date_1h'] is pd.NaT
+    # Next 4 rows contain the starting date (0:00)
+    assert result.iloc[4]['date_1h'] == result.iloc[0]['date']
+    assert result.iloc[5]['date_1h'] == result.iloc[0]['date']
+    assert result.iloc[6]['date_1h'] == result.iloc[0]['date']
+    assert result.iloc[7]['date_1h'] == result.iloc[0]['date']
+    # Next 4 rows contain the next Hourly date original date row 4
+    assert result.iloc[8]['date_1h'] == result.iloc[4]['date']
+
+
+def test_merge_informative_pair_same():
+    data = generate_test_data('15m', 40)
+    informative = generate_test_data('15m', 40)
+
+    result = merge_informative_pair(data, informative, '15m', '15m', ffill=True)
+    assert isinstance(result, pd.DataFrame)
+    assert len(result) == len(data)
+    assert 'date' in result.columns
+    assert result['date'].equals(data['date'])
+    assert 'date_15m' in result.columns
+
+    assert 'open' in result.columns
+    assert 'open_15m' in result.columns
+    assert result['open'].equals(data['open'])
+
+    assert 'close' in result.columns
+    assert 'close_15m' in result.columns
+    assert result['close'].equals(data['close'])
+
+    assert 'volume' in result.columns
+    assert 'volume_15m' in result.columns
+    assert result['volume'].equals(data['volume'])
+
+    # Dates match 1:1
+    assert result['date_15m'].equals(result['date'])
diff --git a/tests/test_arguments.py b/tests/test_arguments.py
index 0052a61d0..e2a1ae53c 100644
--- a/tests/test_arguments.py
+++ b/tests/test_arguments.py
@@ -6,7 +6,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from freqtrade.commands import Arguments
-from freqtrade.commands.cli_options import check_int_positive
+from freqtrade.commands.cli_options import check_int_nonzero, check_int_positive
 
 
 # Parse common command-line-arguments. Used for all tools
@@ -19,64 +19,64 @@ def test_parse_args_none() -> None:
 
 
 def test_parse_args_defaults(mocker) -> None:
-    mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True]))
+    mocker.patch.object(Path, 'is_file', MagicMock(side_effect=[False, True]))
     args = Arguments(['trade']).get_parsed_arg()
-    assert args["config"] == ['config.json']
-    assert args["strategy_path"] is None
-    assert args["datadir"] is None
-    assert args["verbosity"] == 0
+    assert args['config'] == ['config.json']
+    assert args['strategy_path'] is None
+    assert args['datadir'] is None
+    assert args['verbosity'] == 0
 
 
 def test_parse_args_default_userdatadir(mocker) -> None:
-    mocker.patch.object(Path, "is_file", MagicMock(return_value=True))
+    mocker.patch.object(Path, 'is_file', MagicMock(return_value=True))
     args = Arguments(['trade']).get_parsed_arg()
     # configuration defaults to user_data if that is available.
-    assert args["config"] == [str(Path('user_data/config.json'))]
-    assert args["strategy_path"] is None
-    assert args["datadir"] is None
-    assert args["verbosity"] == 0
+    assert args['config'] == [str(Path('user_data/config.json'))]
+    assert args['strategy_path'] is None
+    assert args['datadir'] is None
+    assert args['verbosity'] == 0
 
 
 def test_parse_args_userdatadir(mocker) -> None:
-    mocker.patch.object(Path, "is_file", MagicMock(return_value=True))
+    mocker.patch.object(Path, 'is_file', MagicMock(return_value=True))
     args = Arguments(['trade', '--user-data-dir', 'user_data']).get_parsed_arg()
     # configuration defaults to user_data if that is available.
-    assert args["config"] == [str(Path('user_data/config.json'))]
-    assert args["strategy_path"] is None
-    assert args["datadir"] is None
-    assert args["verbosity"] == 0
+    assert args['config'] == [str(Path('user_data/config.json'))]
+    assert args['strategy_path'] is None
+    assert args['datadir'] is None
+    assert args['verbosity'] == 0
 
 
 def test_parse_args_config() -> None:
     args = Arguments(['trade', '-c', '/dev/null']).get_parsed_arg()
-    assert args["config"] == ['/dev/null']
+    assert args['config'] == ['/dev/null']
 
     args = Arguments(['trade', '--config', '/dev/null']).get_parsed_arg()
-    assert args["config"] == ['/dev/null']
+    assert args['config'] == ['/dev/null']
 
     args = Arguments(['trade', '--config', '/dev/null',
                       '--config', '/dev/zero'],).get_parsed_arg()
-    assert args["config"] == ['/dev/null', '/dev/zero']
+    assert args['config'] == ['/dev/null', '/dev/zero']
 
 
 def test_parse_args_db_url() -> None:
     args = Arguments(['trade', '--db-url', 'sqlite:///test.sqlite']).get_parsed_arg()
-    assert args["db_url"] == 'sqlite:///test.sqlite'
+    assert args['db_url'] == 'sqlite:///test.sqlite'
 
 
 def test_parse_args_verbose() -> None:
     args = Arguments(['trade', '-v']).get_parsed_arg()
-    assert args["verbosity"] == 1
+    assert args['verbosity'] == 1
 
     args = Arguments(['trade', '--verbose']).get_parsed_arg()
-    assert args["verbosity"] == 1
+    assert args['verbosity'] == 1
 
 
 def test_common_scripts_options() -> None:
     args = Arguments(['download-data', '-p', 'ETH/BTC', 'XRP/BTC']).get_parsed_arg()
 
-    assert args["pairs"] == ['ETH/BTC', 'XRP/BTC']
-    assert "func" in args
+    assert args['pairs'] == ['ETH/BTC', 'XRP/BTC']
+    assert 'func' in args
 
 
 def test_parse_args_version() -> None:
@@ -91,7 +91,7 @@ def test_parse_args_invalid() -> None:
 
 def test_parse_args_strategy() -> None:
     args = Arguments(['trade', '--strategy', 'SomeStrategy']).get_parsed_arg()
-    assert args["strategy"] == 'SomeStrategy'
+    assert args['strategy'] == 'SomeStrategy'
 
 
 def test_parse_args_strategy_invalid() -> None:
@@ -101,7 +101,7 @@ def test_parse_args_strategy_invalid() -> None:
 
 def test_parse_args_strategy_path() -> None:
     args = Arguments(['trade', '--strategy-path', '/some/path']).get_parsed_arg()
-    assert args["strategy_path"] == '/some/path'
+    assert args['strategy_path'] == '/some/path'
 
 
 def test_parse_args_strategy_path_invalid() -> None:
@@ -127,13 +127,13 @@ def test_parse_args_backtesting_custom() -> None:
         'SampleStrategy'
         ]
     call_args = Arguments(args).get_parsed_arg()
-    assert call_args["config"] == ['test_conf.json']
-    assert call_args["verbosity"] == 0
-    assert call_args["command"] == 'backtesting'
-    assert call_args["func"] is not None
-    assert call_args["ticker_interval"] == '1m'
-    assert type(call_args["strategy_list"]) is list
-    assert len(call_args["strategy_list"]) == 2
+    assert call_args['config'] == ['test_conf.json']
+    assert call_args['verbosity'] == 0
+    assert call_args['command'] == 'backtesting'
+    assert call_args['func'] is not None
+    assert call_args['timeframe'] == '1m'
+    assert type(call_args['strategy_list']) is list
+    assert len(call_args['strategy_list']) == 2
 
 
 def test_parse_args_hyperopt_custom() -> None:
@@ -144,13 +144,13 @@ def test_parse_args_hyperopt_custom() -> None:
         '--spaces', 'buy'
     ]
     call_args = Arguments(args).get_parsed_arg()
-    assert call_args["config"] == ['test_conf.json']
-    assert call_args["epochs"] == 20
-    assert call_args["verbosity"] == 0
-    assert call_args["command"] == 'hyperopt'
-    assert call_args["spaces"] == ['buy']
-    assert call_args["func"] is not None
-    assert callable(call_args["func"])
+    assert call_args['config'] == ['test_conf.json']
+    assert call_args['epochs'] == 20
+    assert call_args['verbosity'] == 0
+    assert call_args['command'] == 'hyperopt'
+    assert call_args['spaces'] == ['buy']
+    assert call_args['func'] is not None
+    assert callable(call_args['func'])
 
 
 def test_download_data_options() -> None:
@@ -163,10 +163,10 @@ def test_download_data_options() -> None:
     ]
     pargs = Arguments(args).get_parsed_arg()
 
-    assert pargs["pairs_file"] == 'file_with_pairs'
-    assert pargs["datadir"] == 'datadir/directory'
-    assert pargs["days"] == 30
-    assert pargs["exchange"] == 'binance'
+    assert pargs['pairs_file'] == 'file_with_pairs'
+    assert pargs['datadir'] == 'datadir/directory'
+    assert pargs['days'] == 30
+    assert pargs['exchange'] == 'binance'
 
 
 def test_plot_dataframe_options() -> None:
@@ -180,10 +180,10 @@ def test_plot_dataframe_options() -> None:
     ]
     pargs = Arguments(args).get_parsed_arg()
 
-    assert pargs["indicators1"] == ["sma10", "sma100"]
-    assert pargs["indicators2"] == ["macd", "fastd", "fastk"]
-    assert pargs["plot_limit"] == 30
-    assert pargs["pairs"] == ["UNITTEST/BTC"]
+    assert pargs['indicators1'] == ['sma10', 'sma100']
+    assert pargs['indicators2'] == ['macd', 'fastd', 'fastk']
+    assert pargs['plot_limit'] == 30
+    assert pargs['pairs'] == ['UNITTEST/BTC']
 
 
 def test_plot_profit_options() -> None:
@@ -191,66 +191,89 @@ def test_plot_profit_options() -> None:
         'plot-profit',
         '-p', 'UNITTEST/BTC',
         '--trade-source', 'DB',
-        "--db-url", "sqlite:///whatever.sqlite",
+        '--db-url', 'sqlite:///whatever.sqlite',
     ]
     pargs = Arguments(args).get_parsed_arg()
 
-    assert pargs["trade_source"] == "DB"
-    assert pargs["pairs"] == ["UNITTEST/BTC"]
-    assert pargs["db_url"] == "sqlite:///whatever.sqlite"
+    assert pargs['trade_source'] == 'DB'
+    assert pargs['pairs'] == ['UNITTEST/BTC']
+    assert pargs['db_url'] == 'sqlite:///whatever.sqlite'
 
 
 def test_config_notallowed(mocker) -> None:
-    mocker.patch.object(Path, "is_file", MagicMock(return_value=False))
+    mocker.patch.object(Path, 'is_file', MagicMock(return_value=False))
     args = [
         'create-userdir',
     ]
     pargs = Arguments(args).get_parsed_arg()
 
-    assert "config" not in pargs
+    assert 'config' not in pargs
 
     # When file exists:
-    mocker.patch.object(Path, "is_file", MagicMock(return_value=True))
+    mocker.patch.object(Path, 'is_file', MagicMock(return_value=True))
     args = [
         'create-userdir',
     ]
     pargs = Arguments(args).get_parsed_arg()
     # config is not added even if it exists, since create-userdir is in the notallowed list
-    assert "config" not in pargs
+    assert 'config' not in pargs
 
 
 def test_config_notrequired(mocker) -> None:
-    mocker.patch.object(Path, "is_file", MagicMock(return_value=False))
+    mocker.patch.object(Path, 'is_file', MagicMock(return_value=False))
     args = [
         'download-data',
     ]
     pargs = Arguments(args).get_parsed_arg()
 
-    assert pargs["config"] is None
+    assert pargs['config'] is None
 
     # When file exists:
-    mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True]))
+    mocker.patch.object(Path, 'is_file', MagicMock(side_effect=[False, True]))
     args = [
         'download-data',
     ]
     pargs = Arguments(args).get_parsed_arg()
     # config is added if it exists
-    assert pargs["config"] == ['config.json']
+    assert pargs['config'] == ['config.json']
 
 
 def test_check_int_positive() -> None:
-    assert check_int_positive("3") == 3
-    assert check_int_positive("1") == 1
-    assert check_int_positive("100") == 100
+    assert check_int_positive('3') == 3
+    assert check_int_positive('1') == 1
+    assert check_int_positive('100') == 100
 
     with pytest.raises(argparse.ArgumentTypeError):
-        check_int_positive("-2")
+        check_int_positive('-2')
 
     with pytest.raises(argparse.ArgumentTypeError):
-        check_int_positive("0")
+        check_int_positive('0')
 
     with pytest.raises(argparse.ArgumentTypeError):
-        check_int_positive("3.5")
+        check_int_positive(0)
 
     with pytest.raises(argparse.ArgumentTypeError):
-        check_int_positive("DeadBeef")
+        check_int_positive('3.5')
+
+    with pytest.raises(argparse.ArgumentTypeError):
+        check_int_positive('DeadBeef')
+
+
+def test_check_int_nonzero() -> None:
+    assert check_int_nonzero('3') == 3
+    assert check_int_nonzero('1') == 1
+    assert check_int_nonzero('100') == 100
+
+    assert check_int_nonzero('-2') == -2
+
+    with pytest.raises(argparse.ArgumentTypeError):
+        check_int_nonzero('0')
+
+    with pytest.raises(argparse.ArgumentTypeError):
+        check_int_nonzero(0)
+
+    with pytest.raises(argparse.ArgumentTypeError):
+        check_int_nonzero('3.5')
+
+    with pytest.raises(argparse.ArgumentTypeError):
+        check_int_nonzero('DeadBeef')
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index dcb5e2ed0..bebbc1508 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -11,20 +11,19 @@ import pytest
 from jsonschema import ValidationError
 
 from freqtrade.commands import Arguments
-from freqtrade.configuration import (Configuration, check_exchange,
-                                     remove_credentials,
+from freqtrade.configuration import (Configuration, check_exchange, remove_credentials,
                                      validate_config_consistency)
 from freqtrade.configuration.config_validation import validate_config_schema
-from freqtrade.configuration.deprecated_settings import (
-    check_conflicting_settings, process_deprecated_setting,
-    process_temporary_deprecated_settings)
+from freqtrade.configuration.deprecated_settings import (check_conflicting_settings,
+                                                         process_deprecated_setting,
+                                                         process_removed_setting,
+                                                         process_temporary_deprecated_settings)
 from freqtrade.configuration.load_config import load_config_file, log_config_error_range
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
 from freqtrade.exceptions import OperationalException
-from freqtrade.loggers import _set_loggers, setup_logging
+from freqtrade.loggers import _set_loggers, setup_logging, setup_logging_pre
 from freqtrade.state import RunMode
-from tests.conftest import (log_has, log_has_re,
-                            patched_configuration_load_config_file)
+from tests.conftest import log_has, log_has_re, patched_configuration_load_config_file
 
 
 @pytest.fixture(scope="function")
@@ -87,7 +86,7 @@ def test_load_config_file_error_range(default_conf, mocker, caplog) -> None:
     assert isinstance(x, str)
     assert (x == '{"max_open_trades": 1, "stake_currency": "BTC", '
             '"stake_amount": .001, "fiat_display_currency": "USD", '
-            '"ticker_interval": "5m", "dry_run": true, ')
+            '"timeframe": "5m", "dry_run": true, "cance')
 
 
 def test__args_to_config(caplog):
@@ -401,8 +400,8 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) ->
     assert 'datadir' in config
     assert 'user_data_dir' in config
     assert log_has('Using data directory: {} ...'.format(config['datadir']), caplog)
-    assert 'ticker_interval' in config
-    assert not log_has('Parameter -i/--ticker-interval detected ...', caplog)
+    assert 'timeframe' in config
+    assert not log_has('Parameter -i/--timeframe detected ...', caplog)
 
     assert 'position_stacking' not in config
     assert not log_has('Parameter --enable-position-stacking detected ...', caplog)
@@ -448,8 +447,8 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non
     assert log_has('Using user-data directory: {} ...'.format(Path("/tmp/freqtrade")), caplog)
     assert 'user_data_dir' in config
 
-    assert 'ticker_interval' in config
-    assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
+    assert 'timeframe' in config
+    assert log_has('Parameter -i/--timeframe detected ... Using timeframe: 1m ...',
                    caplog)
 
     assert 'position_stacking' in config
@@ -494,8 +493,8 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non
     assert 'pair_whitelist' in config['exchange']
     assert 'datadir' in config
     assert log_has('Using data directory: {} ...'.format(config['datadir']), caplog)
-    assert 'ticker_interval' in config
-    assert log_has('Parameter -i/--ticker-interval detected ... Using ticker_interval: 1m ...',
+    assert 'timeframe' in config
+    assert log_has('Parameter -i/--timeframe detected ... Using timeframe: 1m ...',
                    caplog)
 
     assert 'strategy_list' in config
@@ -665,7 +664,7 @@ def test_set_loggers() -> None:
 
 
 @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows")
-def test_set_loggers_syslog(mocker):
+def test_set_loggers_syslog():
     logger = logging.getLogger()
     orig_handlers = logger.handlers
     logger.handlers = []
@@ -674,14 +673,44 @@ def test_set_loggers_syslog(mocker):
               'logfile': 'syslog:/dev/log',
               }
 
+    setup_logging_pre()
     setup_logging(config)
-    assert len(logger.handlers) == 2
+    assert len(logger.handlers) == 3
     assert [x for x in logger.handlers if type(x) == logging.handlers.SysLogHandler]
     assert [x for x in logger.handlers if type(x) == logging.StreamHandler]
+    assert [x for x in logger.handlers if type(x) == logging.handlers.BufferingHandler]
+    # setting up logging again should NOT cause the loggers to be added a second time.
+    setup_logging(config)
+    assert len(logger.handlers) == 3
     # reset handlers to not break pytest
     logger.handlers = orig_handlers
 
 
+@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows")
+def test_set_loggers_Filehandler(tmpdir):
+    logger = logging.getLogger()
+    orig_handlers = logger.handlers
+    logger.handlers = []
+    logfile = Path(tmpdir) / 'ft_logfile.log'
+    config = {'verbosity': 2,
+              'logfile': str(logfile),
+              }
+
+    setup_logging_pre()
+    setup_logging(config)
+    assert len(logger.handlers) == 3
+    assert [x for x in logger.handlers if type(x) == logging.handlers.RotatingFileHandler]
+    assert [x for x in logger.handlers if type(x) == logging.StreamHandler]
+    assert [x for x in logger.handlers if type(x) == logging.handlers.BufferingHandler]
+    # setting up logging again should NOT cause the loggers to be added a second time.
+    setup_logging(config)
+    assert len(logger.handlers) == 3
+    # reset handlers to not break pytest
+    if logfile.exists:
+        logfile.unlink()
+    logger.handlers = orig_handlers
+
+
 @pytest.mark.skip(reason="systemd is not installed on every system, so we're not testing this.")
 def test_set_loggers_journald(mocker):
     logger = logging.getLogger()
@@ -727,7 +756,10 @@ def test_set_logfile(default_conf, mocker):
     assert validated_conf['logfile'] == "test_file.log"
     f = Path("test_file.log")
     assert f.is_file()
-    f.unlink()
+    try:
+        f.unlink()
+    except Exception:
+        pass
 
 
 def test_load_config_warn_forcebuy(default_conf, mocker, caplog) -> None:
@@ -809,6 +841,21 @@ def test_validate_edge(edge_conf):
     validate_config_consistency(edge_conf)
 
 
+def test_validate_edge2(edge_conf):
+    edge_conf.update({"ask_strategy": {
+        "use_sell_signal": True,
+    }})
+    # Passes test
+    validate_config_consistency(edge_conf)
+
+    edge_conf.update({"ask_strategy": {
+        "use_sell_signal": False,
+    }})
+    with pytest.raises(OperationalException, match="Edge requires `use_sell_signal` to be True, "
+                       "otherwise no sells will happen."):
+        validate_config_consistency(edge_conf)
+
+
 def test_validate_whitelist(default_conf):
     default_conf['runmode'] = RunMode.DRY_RUN
     # Test regular case - has whitelist and uses StaticPairlist
@@ -833,6 +880,25 @@ def test_validate_whitelist(default_conf):
     validate_config_consistency(conf)
 
 
+@pytest.mark.parametrize('protconf,expected', [
+    ([], None),
+    ([{"method": "StoplossGuard", "lookback_period": 2000, "stop_duration_candles": 10}], None),
+    ([{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 10}], None),
+    ([{"method": "StoplossGuard", "lookback_period_candles": 20, "lookback_period": 2000,
+       "stop_duration": 10}], r'Protections must specify either `lookback_period`.*'),
+    ([{"method": "StoplossGuard", "lookback_period": 20, "stop_duration": 10,
+       "stop_duration_candles": 10}], r'Protections must specify either `stop_duration`.*'),
+])
+def test_validate_protections(default_conf, protconf, expected):
+    conf = deepcopy(default_conf)
+    conf['protections'] = protconf
+    if expected:
+        with pytest.raises(OperationalException, match=expected):
+            validate_config_consistency(conf)
+    else:
+        validate_config_consistency(conf)
+
+
 def test_load_config_test_comments() -> None:
     """
     Load config with comments
@@ -871,6 +937,14 @@ def test_load_config_default_exchange_name(all_conf) -> None:
         validate_config_schema(all_conf)
 
 
+def test_load_config_stoploss_exchange_limit_ratio(all_conf) -> None:
+    all_conf['order_types']['stoploss_on_exchange_limit_ratio'] = 1.15
+
+    with pytest.raises(ValidationError,
+                       match=r"1.15 is greater than the maximum"):
+        validate_config_schema(all_conf)
+
+
 @pytest.mark.parametrize("keys", [("exchange", "sandbox", False),
                                   ("exchange", "key", ""),
                                   ("exchange", "secret", ""),
@@ -997,7 +1071,7 @@ def test_pairlist_resolving_fallback(mocker):
 
     args = Arguments(arglist).get_parsed_arg()
     # Fix flaky tests if config.json exists
-    args["config"] = None
+    args['config'] = None
 
     configuration = Configuration(args, RunMode.OTHER)
     config = configuration.get_config()
@@ -1007,13 +1081,11 @@ def test_pairlist_resolving_fallback(mocker):
     assert config['datadir'] == Path.cwd() / "user_data/data/binance"
 
 
+@pytest.mark.skip(reason='Currently no deprecated / moved sections')
+# The below is kept as a sample for the future.
 @pytest.mark.parametrize("setting", [
         ("ask_strategy", "use_sell_signal", True,
          "experimental", "use_sell_signal", False),
-        ("ask_strategy", "sell_profit_only", False,
-         "experimental", "sell_profit_only", True),
-        ("ask_strategy", "ignore_roi_if_buy_signal", False,
-         "experimental", "ignore_roi_if_buy_signal", True),
     ])
 def test_process_temporary_deprecated_settings(mocker, default_conf, setting, caplog):
     patched_configuration_load_config_file(mocker, default_conf)
@@ -1043,7 +1115,27 @@ def test_process_temporary_deprecated_settings(mocker, default_conf, setting, ca
     assert default_conf[setting[0]][setting[1]] == setting[5]
 
 
-def test_process_deprecated_setting_edge(mocker, edge_conf, caplog):
+@pytest.mark.parametrize("setting", [
+        ("experimental", "use_sell_signal", False),
+        ("experimental", "sell_profit_only", True),
+        ("experimental", "ignore_roi_if_buy_signal", True),
+    ])
+def test_process_removed_settings(mocker, default_conf, setting):
+    patched_configuration_load_config_file(mocker, default_conf)
+
+    # Create sections for new and deprecated settings
+    # (they may not exist in the config)
+    default_conf[setting[0]] = {}
+    # Assign removed setting
+    default_conf[setting[0]][setting[1]] = setting[2]
+
+    # New and deprecated settings are conflicting ones
+    with pytest.raises(OperationalException,
+                       match=r'Setting .* has been moved'):
+        process_temporary_deprecated_settings(default_conf)
+
+
+def test_process_deprecated_setting_edge(mocker, edge_conf):
     patched_configuration_load_config_file(mocker, edge_conf)
     edge_conf.update({'edge': {
         'enabled': True,
@@ -1140,3 +1232,49 @@ def test_process_deprecated_setting(mocker, default_conf, caplog):
                                'sectionB', 'deprecated_setting')
     assert not log_has_re('DEPRECATED', caplog)
     assert default_conf['sectionA']['new_setting'] == 'valA'
+
+
+def test_process_removed_setting(mocker, default_conf, caplog):
+    patched_configuration_load_config_file(mocker, default_conf)
+
+    # Create sections for new and deprecated settings
+    # (they may not exist in the config)
+    default_conf['sectionA'] = {}
+    default_conf['sectionB'] = {}
+    # Assign new setting
+    default_conf['sectionB']['somesetting'] = 'valA'
+
+    # Only new setting exists (nothing should happen)
+    process_removed_setting(default_conf,
+                            'sectionA', 'somesetting',
+                            'sectionB', 'somesetting')
+    # Assign removed setting
+    default_conf['sectionA']['somesetting'] = 'valB'
+
+    with pytest.raises(OperationalException,
+                       match=r"Setting .* has been moved"):
+        process_removed_setting(default_conf,
+                                'sectionA', 'somesetting',
+                                'sectionB', 'somesetting')
+
+
+def test_process_deprecated_ticker_interval(mocker, default_conf, caplog):
+    message = "DEPRECATED: Please use 'timeframe' instead of 'ticker_interval."
+    config = deepcopy(default_conf)
+    process_temporary_deprecated_settings(config)
+    assert not log_has(message, caplog)
+
+    del config['timeframe']
+    config['ticker_interval'] = '15m'
+    process_temporary_deprecated_settings(config)
+    assert log_has(message, caplog)
+    assert config['ticker_interval'] == '15m'
+
+    config = deepcopy(default_conf)
+    # Have both timeframe and ticker interval in config
+    # Can also happen when using ticker_interval in configuration, and --timeframe as cli argument
+    config['timeframe'] = '5m'
+    config['ticker_interval'] = '4h'
+    with pytest.raises(OperationalException,
+                       match=r"Both 'timeframe' and 'ticker_interval' detected."):
+        process_temporary_deprecated_settings(config)
diff --git a/tests/test_directory_operations.py b/tests/test_directory_operations.py
index 71c91549f..a8058c514 100644
--- a/tests/test_directory_operations.py
+++ b/tests/test_directory_operations.py
@@ -4,8 +4,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
-from freqtrade.configuration.directory_operations import (copy_sample_files,
-                                                          create_datadir,
+from freqtrade.configuration.directory_operations import (copy_sample_files, create_datadir,
                                                           create_userdata_dir)
 from freqtrade.exceptions import OperationalException
 from tests.conftest import log_has, log_has_re
diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index 487e3a60e..12be5ae8b 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -9,23 +9,24 @@ from unittest.mock import ANY, MagicMock, PropertyMock
 
 import arrow
 import pytest
-import requests
 
-from freqtrade.constants import (CANCEL_REASON, MATH_CLOSE_PREC,
-                                 UNLIMITED_STAKE_AMOUNT)
-from freqtrade.exceptions import (DependencyException, InvalidOrderException,
-                                  OperationalException, PricingError,
+from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT
+from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError,
+                                  InvalidOrderException, OperationalException, PricingError,
                                   TemporaryError)
 from freqtrade.freqtradebot import FreqtradeBot
-from freqtrade.persistence import Trade
+from freqtrade.persistence import Order, PairLocks, Trade
+from freqtrade.persistence.models import PairLock
 from freqtrade.rpc import RPCMessageType
 from freqtrade.state import RunMode, State
 from freqtrade.strategy.interface import SellCheckTuple, SellType
 from freqtrade.worker import Worker
-from tests.conftest import (create_mock_trades, get_patched_freqtradebot,
-                            get_patched_worker, log_has, log_has_re,
-                            patch_edge, patch_exchange, patch_get_signal,
+from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker,
+                            log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal,
                             patch_wallet, patch_whitelist)
+from tests.conftest_trades import (MOCK_TRADE_COUNT, mock_order_1, mock_order_2, mock_order_2_sell,
+                                   mock_order_3, mock_order_3_sell, mock_order_4,
+                                   mock_order_5_stoploss, mock_order_6_sell)
 
 
 def patch_RPCManager(mocker) -> MagicMock:
@@ -65,7 +66,7 @@ def test_process_stopped(mocker, default_conf) -> None:
 
 
 def test_bot_cleanup(mocker, default_conf, caplog) -> None:
-    mock_cleanup = mocker.patch('freqtrade.persistence.cleanup')
+    mock_cleanup = mocker.patch('freqtrade.freqtradebot.cleanup_db')
     coo_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cancel_all_open_orders')
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     freqtrade.cleanup()
@@ -171,7 +172,7 @@ def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None:
                         (True, 0.0027, 3, 0.5, [0.001, 0.001, 0.000673]),
                         (True, 0.0022, 3, 1, [0.001, 0.001, 0.0]),
                         ])
-def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order,
+def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order_open,
                                       amend_last, wallet, max_open, lsamr, expected) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -179,7 +180,7 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee
     )
     default_conf['dry_run_wallet'] = wallet
@@ -192,6 +193,7 @@ def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_b
     for i in range(0, max_open):
 
         if expected[i] is not None:
+            limit_buy_order_open['id'] = str(i)
             result = freqtrade.get_trade_stake_amount('ETH/BTC')
             assert pytest.approx(result) == expected[i]
             freqtrade.execute_buy('ETH/BTC', result)
@@ -217,13 +219,13 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None:
                         (0.50, 0.0025),
                         ])
 def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_ratio, result1,
-                                                 limit_buy_order, fee, mocker) -> None:
+                                                 limit_buy_order_open, fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee
     )
 
@@ -304,7 +306,6 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf
             'ask': buy_price * 0.79,
             'last': buy_price * 0.79
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         get_fee=fee,
     )
     #############################################
@@ -321,7 +322,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf
 
     # stoploss shoud be hit
     assert freqtrade.handle_trade(trade) is True
-    assert log_has('Executing Sell for NEO/BTC. Reason: SellType.STOP_LOSS', caplog)
+    assert log_has('Executing Sell for NEO/BTC. Reason: stop_loss', caplog)
     assert trade.sell_reason == SellType.STOP_LOSS.value
 
 
@@ -344,7 +345,6 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee,
             'ask': buy_price * 0.85,
             'last': buy_price * 0.85
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         get_fee=fee,
     )
     #############################################
@@ -363,8 +363,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee,
     assert freqtrade.handle_trade(trade) is False
 
 
-def test_total_open_trades_stakes(mocker, default_conf, ticker,
-                                  limit_buy_order, fee) -> None:
+def test_total_open_trades_stakes(mocker, default_conf, ticker, fee) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     default_conf['stake_amount'] = 0.00098751
@@ -372,7 +371,6 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker,
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -535,7 +533,6 @@ def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> Non
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         get_fee=fee,
     )
 
@@ -569,7 +566,6 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order,
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -579,11 +575,11 @@ def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order,
         freqtrade.create_trade('ETH/BTC')
 
 
-def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order,
+def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order_open,
                                      fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
-    buy_mock = MagicMock(return_value={'id': limit_buy_order['id']})
+    buy_mock = MagicMock(return_value=limit_buy_order_open)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
@@ -596,14 +592,14 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order,
 
     freqtrade.create_trade('ETH/BTC')
     rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount']
-    assert rate * amount >= default_conf['stake_amount']
+    assert rate * amount <= default_conf['stake_amount']
 
 
-def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order,
+def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order_open,
                                              fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
-    buy_mock = MagicMock(return_value={'id': limit_buy_order['id']})
+    buy_mock = MagicMock(return_value=limit_buy_order_open)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
@@ -619,14 +615,14 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord
     assert not freqtrade.create_trade('ETH/BTC')
 
 
-def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order,
-                                    fee, markets, mocker) -> None:
+def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order_open,
+                                    fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_balance=MagicMock(return_value=default_conf['stake_amount']),
         get_fee=fee,
     )
@@ -640,14 +636,14 @@ def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order,
     assert freqtrade.get_trade_stake_amount('ETH/BTC') == 0
 
 
-def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order, fee,
+def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_open, fee,
                                        mocker, caplog) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
 
@@ -682,6 +678,32 @@ def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_o
     assert log_has("Active pair whitelist is empty.", caplog)
 
 
+@pytest.mark.usefixtures("init_persistence")
+def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, fee,
+                                         mocker, caplog) -> None:
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        fetch_ticker=ticker,
+        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        get_fee=fee,
+    )
+    freqtrade = FreqtradeBot(default_conf)
+    patch_get_signal(freqtrade)
+    n = freqtrade.enter_positions()
+    message = r"Global pairlock active until.* Not creating new trades."
+    n = freqtrade.enter_positions()
+    # 0 trades, but it's not because of pairlock.
+    assert n == 0
+    assert not log_has_re(message, caplog)
+
+    PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because')
+    n = freqtrade.enter_positions()
+    assert n == 0
+    assert log_has_re(message, caplog)
+
+
 def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
     default_conf['dry_run'] = True
 
@@ -703,7 +725,7 @@ def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
 
 @pytest.mark.parametrize("max_open", range(0, 5))
 @pytest.mark.parametrize("tradable_balance_ratio,modifier", [(1.0, 1), (0.99, 0.8), (0.5, 0.5)])
-def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker,
+def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, limit_buy_order_open,
                                        max_open, tradable_balance_ratio, modifier) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -714,7 +736,7 @@ def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker,
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': "12355555"}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -728,14 +750,14 @@ def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker,
     assert len(trades) == max(int(max_open * modifier), 0)
 
 
-def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None:
+def test_create_trades_preopen(default_conf, ticker, fee, mocker, limit_buy_order_open) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     default_conf['max_open_trades'] = 4
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': "12355555"}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -746,6 +768,8 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None:
     freqtrade.execute_buy('NEO/BTC', default_conf['stake_amount'])
 
     assert len(Trade.get_open_trades()) == 2
+    # Change order_id for new orders
+    limit_buy_order_open['id'] = '123444'
 
     # Create 2 new trades using create_trades
     assert freqtrade.create_trade('ETH/BTC')
@@ -755,15 +779,15 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None:
     assert len(trades) == 4
 
 
-def test_process_trade_creation(default_conf, ticker, limit_buy_order,
+def test_process_trade_creation(default_conf, ticker, limit_buy_order, limit_buy_order_open,
                                 fee, mocker, caplog) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
-        get_order=MagicMock(return_value=limit_buy_order),
+        buy=MagicMock(return_value=limit_buy_order_open),
+        fetch_order=MagicMock(return_value=limit_buy_order),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -783,7 +807,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order,
     assert trade.open_date is not None
     assert trade.exchange == 'bittrex'
     assert trade.open_rate == 0.00001098
-    assert trade.amount == 91.07468123861567
+    assert trade.amount == 91.07468123
 
     assert log_has(
         'Buy signal found: about create a new trade with stake_amount: 0.001 ...', caplog
@@ -825,14 +849,14 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None:
     assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status']
 
 
-def test_process_trade_handling(default_conf, ticker, limit_buy_order, fee, mocker) -> None:
+def test_process_trade_handling(default_conf, ticker, limit_buy_order_open, fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
-        get_order=MagicMock(return_value=limit_buy_order),
+        buy=MagicMock(return_value=limit_buy_order_open),
+        fetch_order=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -859,7 +883,7 @@ def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order,
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
         buy=MagicMock(return_value={'id': limit_buy_order['id']}),
-        get_order=MagicMock(return_value=limit_buy_order),
+        fetch_order=MagicMock(return_value=limit_buy_order),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -912,6 +936,7 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None:
         refresh_latest_ohlcv=refresh_mock,
     )
     inf_pairs = MagicMock(return_value=[("BTC/ETH", '1m'), ("ETH/USDT", "1h")])
+    mocker.patch('freqtrade.strategy.interface.IStrategy.get_signal', return_value=(False, False))
     mocker.patch('time.sleep', return_value=None)
 
     freqtrade = FreqtradeBot(default_conf)
@@ -924,7 +949,7 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None:
     assert refresh_mock.call_count == 1
     assert ("BTC/ETH", "1m") in refresh_mock.call_args[0][0]
     assert ("ETH/USDT", "1h") in refresh_mock.call_args[0][0]
-    assert ("ETH/BTC", default_conf["ticker_interval"]) in refresh_mock.call_args[0][0]
+    assert ("ETH/BTC", default_conf["timeframe"]) in refresh_mock.call_args[0][0]
 
 
 @pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [
@@ -953,6 +978,7 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None:
 ])
 def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid,
                       last, last_ab, expected) -> None:
+    caplog.set_level(logging.DEBUG)
     default_conf['bid_strategy']['ask_last_balance'] = last_ab
     default_conf['bid_strategy']['price_side'] = side
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
@@ -970,10 +996,11 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid,
     assert not log_has("Using cached buy rate for ETH/BTC.", caplog)
 
 
-def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None:
+def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     freqtrade = FreqtradeBot(default_conf)
+    freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=False)
     stake_amount = 2
     bid = 0.11
     buy_rate_mock = MagicMock(return_value=bid)
@@ -982,7 +1009,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None:
         get_buy_rate=buy_rate_mock,
         _get_min_pair_stake_amount=MagicMock(return_value=1)
     )
-    buy_mm = MagicMock(return_value={'id': limit_buy_order['id']})
+    buy_mm = MagicMock(return_value=limit_buy_order_open)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=MagicMock(return_value={
@@ -995,37 +1022,49 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None:
     )
     pair = 'ETH/BTC'
 
+    assert not freqtrade.execute_buy(pair, stake_amount)
+    assert buy_rate_mock.call_count == 1
+    assert buy_mm.call_count == 0
+    assert freqtrade.strategy.confirm_trade_entry.call_count == 1
+    buy_rate_mock.reset_mock()
+
+    limit_buy_order_open['id'] = '22'
+    freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True)
     assert freqtrade.execute_buy(pair, stake_amount)
     assert buy_rate_mock.call_count == 1
     assert buy_mm.call_count == 1
     call_args = buy_mm.call_args_list[0][1]
     assert call_args['pair'] == pair
     assert call_args['rate'] == bid
-    assert call_args['amount'] == stake_amount / bid
+    assert call_args['amount'] == round(stake_amount / bid, 8)
+    buy_rate_mock.reset_mock()
 
     # Should create an open trade with an open order id
     # As the order is not fulfilled yet
     trade = Trade.query.first()
     assert trade
     assert trade.is_open is True
-    assert trade.open_order_id == limit_buy_order['id']
+    assert trade.open_order_id == '22'
 
     # Test calling with price
+    limit_buy_order_open['id'] = '33'
     fix_price = 0.06
     assert freqtrade.execute_buy(pair, stake_amount, fix_price)
     # Make sure get_buy_rate wasn't called again
-    assert buy_rate_mock.call_count == 1
+    assert buy_rate_mock.call_count == 0
 
     assert buy_mm.call_count == 2
     call_args = buy_mm.call_args_list[1][1]
     assert call_args['pair'] == pair
     assert call_args['rate'] == fix_price
-    assert call_args['amount'] == stake_amount / fix_price
+    assert call_args['amount'] == round(stake_amount / fix_price, 8)
 
     # In case of closed order
     limit_buy_order['status'] = 'closed'
     limit_buy_order['price'] = 10
     limit_buy_order['cost'] = 100
+    limit_buy_order['id'] = '444'
+
     mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order))
     assert freqtrade.execute_buy(pair, stake_amount)
     trade = Trade.query.all()[2]
@@ -1041,11 +1080,12 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None:
     limit_buy_order['remaining'] = 10.00
     limit_buy_order['price'] = 0.5
     limit_buy_order['cost'] = 40.495905365
+    limit_buy_order['id'] = '555'
     mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order))
     assert freqtrade.execute_buy(pair, stake_amount)
     trade = Trade.query.all()[3]
     assert trade
-    assert trade.open_order_id is None
+    assert trade.open_order_id == '555'
     assert trade.open_rate == 0.5
     assert trade.stake_amount == 40.495905365
 
@@ -1056,15 +1096,57 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None:
     limit_buy_order['remaining'] = 90.99181073
     limit_buy_order['price'] = 0.5
     limit_buy_order['cost'] = 0.0
+    limit_buy_order['id'] = '66'
     mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order))
     assert not freqtrade.execute_buy(pair, stake_amount)
 
+    # Fail to get price...
+    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_buy_rate', MagicMock(return_value=0.0))
+
+    with pytest.raises(PricingError, match="Could not determine buy price."):
+        freqtrade.execute_buy(pair, stake_amount)
+
+
+def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) -> None:
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mocker.patch.multiple(
+        'freqtrade.freqtradebot.FreqtradeBot',
+        get_buy_rate=MagicMock(return_value=0.11),
+        _get_min_pair_stake_amount=MagicMock(return_value=1)
+    )
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        fetch_ticker=MagicMock(return_value={
+            'bid': 0.00001172,
+            'ask': 0.00001173,
+            'last': 0.00001172
+        }),
+        buy=MagicMock(return_value=limit_buy_order),
+        get_fee=fee,
+    )
+    stake_amount = 2
+    pair = 'ETH/BTC'
+
+    freqtrade.strategy.confirm_trade_entry = MagicMock(side_effect=ValueError)
+    assert freqtrade.execute_buy(pair, stake_amount)
+
+    limit_buy_order['id'] = '222'
+    freqtrade.strategy.confirm_trade_entry = MagicMock(side_effect=Exception)
+    assert freqtrade.execute_buy(pair, stake_amount)
+
+    limit_buy_order['id'] = '2223'
+    freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True)
+    assert freqtrade.execute_buy(pair, stake_amount)
+
+    freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=False)
+    assert not freqtrade.execute_buy(pair, stake_amount)
+
 
 def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True))
-    mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[])
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount',
                  return_value=limit_buy_order['amount'])
@@ -1126,7 +1208,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
     trade.stoploss_order_id = 100
 
     hanging_stoploss_order = MagicMock(return_value={'status': 'open'})
-    mocker.patch('freqtrade.exchange.Exchange.get_order', hanging_stoploss_order)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', hanging_stoploss_order)
 
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
     assert trade.stoploss_order_id == 100
@@ -1139,7 +1221,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
     trade.stoploss_order_id = 100
 
     canceled_stoploss_order = MagicMock(return_value={'status': 'canceled'})
-    mocker.patch('freqtrade.exchange.Exchange.get_order', canceled_stoploss_order)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', canceled_stoploss_order)
     stoploss.reset_mock()
 
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
@@ -1158,32 +1240,34 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
     assert trade
 
     stoploss_order_hit = MagicMock(return_value={
+        'id': 100,
         'status': 'closed',
         'type': 'stop_loss_limit',
         'price': 3,
         'average': 2,
         'amount': limit_buy_order['amount'],
     })
-    mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hit)
     assert freqtrade.handle_stoploss_on_exchange(trade) is True
-    assert log_has('STOP_LOSS_LIMIT is hit for {}.'.format(trade), caplog)
+    assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog)
     assert trade.stoploss_order_id is None
     assert trade.is_open is False
 
     mocker.patch(
         'freqtrade.exchange.Exchange.stoploss',
-        side_effect=DependencyException()
+        side_effect=ExchangeError()
     )
     trade.is_open = True
     freqtrade.handle_stoploss_on_exchange(trade)
     assert log_has('Unable to place a stoploss order on exchange.', caplog)
     assert trade.stoploss_order_id is None
 
-    # Fifth case: get_order returns InvalidOrder
+    # Fifth case: fetch_order returns InvalidOrder
     # It should try to add stoploss order
     trade.stoploss_order_id = 100
     stoploss.reset_mock()
-    mocker.patch('freqtrade.exchange.Exchange.get_order', side_effect=InvalidOrderException())
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order',
+                 side_effect=InvalidOrderException())
     mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss)
     freqtrade.handle_stoploss_on_exchange(trade)
     assert stoploss.call_count == 1
@@ -1193,7 +1277,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog,
     trade.stoploss_order_id = None
     trade.is_open = False
     stoploss.reset_mock()
-    mocker.patch('freqtrade.exchange.Exchange.get_order')
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order')
     mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss)
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
     assert stoploss.call_count == 0
@@ -1214,8 +1298,8 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog,
         buy=MagicMock(return_value={'id': limit_buy_order['id']}),
         sell=MagicMock(return_value={'id': limit_sell_order['id']}),
         get_fee=fee,
-        get_order=MagicMock(return_value={'status': 'canceled'}),
-        stoploss=MagicMock(side_effect=DependencyException()),
+        fetch_stoploss_order=MagicMock(return_value={'status': 'canceled', 'id': 100}),
+        stoploss=MagicMock(side_effect=ExchangeError()),
     )
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
@@ -1234,7 +1318,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog,
 
 
 def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee,
-                                             limit_buy_order, limit_sell_order):
+                                             limit_buy_order_open, limit_sell_order):
     rpc_mock = patch_RPCManager(mocker)
     patch_exchange(mocker)
     sell_mock = MagicMock(return_value={'id': limit_sell_order['id']})
@@ -1245,10 +1329,10 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee,
             'ask': 0.00001173,
             'last': 0.00001172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         sell=sell_mock,
         get_fee=fee,
-        get_order=MagicMock(return_value={'status': 'canceled'}),
+        fetch_order=MagicMock(return_value={'status': 'canceled'}),
         stoploss=MagicMock(side_effect=InvalidOrderException()),
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -1258,7 +1342,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee,
     freqtrade.enter_positions()
     trade = Trade.query.first()
     caplog.clear()
-    freqtrade.create_stoploss_order(trade, 200, 199)
+    freqtrade.create_stoploss_order(trade, 200)
     assert trade.stoploss_order_id is None
     assert trade.sell_reason == SellType.EMERGENCY_SELL.value
     assert log_has("Unable to place a stoploss order on exchange. ", caplog)
@@ -1276,7 +1360,46 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee,
     assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market'
 
 
-def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog,
+def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, fee,
+                                                  limit_buy_order_open, limit_sell_order):
+    sell_mock = MagicMock(return_value={'id': limit_sell_order['id']})
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+
+    mock_insuf = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_insufficient_funds')
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        fetch_ticker=MagicMock(return_value={
+            'bid': 0.00001172,
+            'ask': 0.00001173,
+            'last': 0.00001172
+        }),
+        buy=MagicMock(return_value=limit_buy_order_open),
+        sell=sell_mock,
+        get_fee=fee,
+        fetch_order=MagicMock(return_value={'status': 'canceled'}),
+        stoploss=MagicMock(side_effect=InsufficientFundsError()),
+    )
+    patch_get_signal(freqtrade)
+    freqtrade.strategy.order_types['stoploss_on_exchange'] = True
+
+    freqtrade.enter_positions()
+    trade = Trade.query.first()
+    caplog.clear()
+    freqtrade.create_stoploss_order(trade, 200)
+    # stoploss_orderid was empty before
+    assert trade.stoploss_order_id is None
+    assert mock_insuf.call_count == 1
+    mock_insuf.reset_mock()
+
+    trade.stoploss_order_id = 'stoploss_orderid'
+    freqtrade.create_stoploss_order(trade, 200)
+    # No change to stoploss-orderid
+    assert trade.stoploss_order_id == 'stoploss_orderid'
+    assert mock_insuf.call_count == 1
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee,
                                               limit_buy_order, limit_sell_order) -> None:
     # When trailing stoploss is set
     stoploss = MagicMock(return_value={'id': 13434334})
@@ -1331,7 +1454,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog,
         }
     })
 
-    mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hanging)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hanging)
 
     # stoploss initially at 5%
     assert freqtrade.handle_trade(trade) is False
@@ -1345,8 +1468,8 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog,
     }))
 
     cancel_order_mock = MagicMock()
-    stoploss_order_mock = MagicMock()
-    mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order_mock)
+    stoploss_order_mock = MagicMock(return_value={'id': 13434334})
+    mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', cancel_order_mock)
     mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss_order_mock)
 
     # stoploss should not be updated as the interval is 60 seconds
@@ -1364,7 +1487,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog,
     assert freqtrade.handle_stoploss_on_exchange(trade) is False
 
     cancel_order_mock.assert_called_once_with(100, 'ETH/BTC')
-    stoploss_order_mock.assert_called_once_with(amount=85.32423208191126,
+    stoploss_order_mock.assert_called_once_with(amount=85.32423208,
                                                 pair='ETH/BTC',
                                                 order_types=freqtrade.strategy.order_types,
                                                 stop_price=0.00002346 * 0.95)
@@ -1429,8 +1552,9 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c
             'stopPrice': '0.1'
         }
     }
-    mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException())
-    mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hanging)
+    mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order',
+                 side_effect=InvalidOrderException())
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hanging)
     freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging)
     assert log_has_re(r"Could not cancel stoploss order abcd for pair ETH/BTC.*", caplog)
 
@@ -1439,8 +1563,8 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c
 
     # Fail creating stoploss order
     caplog.clear()
-    cancel_mock = mocker.patch("freqtrade.exchange.Exchange.cancel_order", MagicMock())
-    mocker.patch("freqtrade.exchange.Exchange.stoploss", side_effect=DependencyException())
+    cancel_mock = mocker.patch("freqtrade.exchange.Exchange.cancel_stoploss_order", MagicMock())
+    mocker.patch("freqtrade.exchange.Exchange.stoploss", side_effect=ExchangeError())
     freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging)
     assert cancel_mock.call_count == 1
     assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog)
@@ -1510,7 +1634,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
         }
     })
 
-    mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hanging)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_order_hanging)
 
     # stoploss initially at 20% as edge dictated it.
     assert freqtrade.handle_trade(trade) is False
@@ -1519,7 +1643,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
 
     cancel_order_mock = MagicMock()
     stoploss_order_mock = MagicMock()
-    mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order_mock)
+    mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order', cancel_order_mock)
     mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss_order_mock)
 
     # price goes down 5%
@@ -1551,7 +1675,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog,
     # stoploss should be set to 1% as trailing is on
     assert trade.stop_loss == 0.00002346 * 0.99
     cancel_order_mock.assert_called_once_with(100, 'NEO/BTC')
-    stoploss_order_mock.assert_called_once_with(amount=2132892.491467577,
+    stoploss_order_mock.assert_called_once_with(amount=2132892.49146757,
                                                 pair='NEO/BTC',
                                                 order_types=freqtrade.strategy.order_types,
                                                 stop_price=0.00002346 * 0.99)
@@ -1587,7 +1711,7 @@ def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True))
-    mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[])
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount',
                  return_value=limit_buy_order['amount'])
@@ -1611,11 +1735,12 @@ def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None:
 
 def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order)
 
     trade = MagicMock()
     trade.open_order_id = None
     trade.open_fee = 0.001
+    trade.pair = 'ETH/BTC'
     trades = [trade]
 
     # Test raise of DependencyException exception
@@ -1625,14 +1750,14 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog)
     )
     n = freqtrade.exit_positions(trades)
     assert n == 0
-    assert log_has('Unable to sell trade: ', caplog)
+    assert log_has('Unable to sell trade ETH/BTC: ', caplog)
 
 
 def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True))
-    mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order)
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[])
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount',
                  return_value=limit_buy_order['amount'])
@@ -1645,8 +1770,10 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No
         open_date=arrow.utcnow().datetime,
         amount=11,
     )
+    assert not freqtrade.update_trade_state(trade, None)
+    assert log_has_re(r'Orderid for trade .* is empty.', caplog)
     # Add datetime explicitly since sqlalchemy defaults apply only once written to database
-    freqtrade.update_trade_state(trade)
+    freqtrade.update_trade_state(trade, '123')
     # Test amount not modified by fee-logic
     assert not log_has_re(r'Applying fee to .*', caplog)
     assert trade.open_order_id is None
@@ -1656,14 +1783,14 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81)
     assert trade.amount != 90.81
     # test amount modified by fee-logic
-    freqtrade.update_trade_state(trade)
+    freqtrade.update_trade_state(trade, '123')
     assert trade.amount == 90.81
     assert trade.open_order_id is None
 
     trade.is_open = True
     trade.open_order_id = None
     # Assert we call handle_trade() if trade is feasible for execution
-    freqtrade.update_trade_state(trade)
+    freqtrade.update_trade_state(trade, '123')
 
     assert log_has_re('Found open order for.*', caplog)
 
@@ -1671,8 +1798,8 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No
 def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee,
                                           mocker):
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    # get_order should not be called!!
-    mocker.patch('freqtrade.exchange.Exchange.get_order', MagicMock(side_effect=ValueError))
+    # fetch_order should not be called!!
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError))
     patch_exchange(mocker)
     Trade.session = MagicMock()
     amount = sum(x['amount'] for x in trades_for_order)
@@ -1682,12 +1809,13 @@ def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_
         amount=amount,
         exchange='binance',
         open_rate=0.245441,
+        open_date=arrow.utcnow().datetime,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_order_id="123456",
         is_open=True,
     )
-    freqtrade.update_trade_state(trade, limit_buy_order)
+    freqtrade.update_trade_state(trade, '123456', limit_buy_order)
     assert trade.amount != amount
     assert trade.amount == limit_buy_order['amount']
 
@@ -1696,8 +1824,8 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_
                                                        limit_buy_order, mocker, caplog):
     trades_for_order[0]['amount'] = limit_buy_order['amount'] + 1e-14
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    # get_order should not be called!!
-    mocker.patch('freqtrade.exchange.Exchange.get_order', MagicMock(side_effect=ValueError))
+    # fetch_order should not be called!!
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError))
     patch_exchange(mocker)
     Trade.session = MagicMock()
     amount = sum(x['amount'] for x in trades_for_order)
@@ -1709,11 +1837,11 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_
         open_rate=0.245441,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
-        open_order_id="123456",
+        open_order_id='123456',
         is_open=True,
         open_date=arrow.utcnow().datetime,
     )
-    freqtrade.update_trade_state(trade, limit_buy_order)
+    freqtrade.update_trade_state(trade, '123456', limit_buy_order)
     assert trade.amount != amount
     assert trade.amount == limit_buy_order['amount']
     assert log_has_re(r'Applying fee on amount for .*', caplog)
@@ -1722,7 +1850,7 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_
 def test_update_trade_state_exception(mocker, default_conf,
                                       limit_buy_order, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=limit_buy_order)
 
     trade = MagicMock()
     trade.open_order_id = '123'
@@ -1733,13 +1861,13 @@ def test_update_trade_state_exception(mocker, default_conf,
         'freqtrade.freqtradebot.FreqtradeBot.get_real_amount',
         side_effect=DependencyException()
     )
-    freqtrade.update_trade_state(trade)
+    freqtrade.update_trade_state(trade, trade.open_order_id)
     assert log_has('Could not update trade amount: ', caplog)
 
 
 def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    mocker.patch('freqtrade.exchange.Exchange.get_order',
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order',
                  MagicMock(side_effect=InvalidOrderException))
 
     trade = MagicMock()
@@ -1748,15 +1876,16 @@ def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None
 
     # Test raise of OperationalException exception
     grm_mock = mocker.patch("freqtrade.freqtradebot.FreqtradeBot.get_real_amount", MagicMock())
-    freqtrade.update_trade_state(trade)
+    freqtrade.update_trade_state(trade, trade.open_order_id)
     assert grm_mock.call_count == 0
     assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog)
 
 
-def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order, mocker):
+def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_open,
+                                 limit_sell_order, mocker):
     mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order)
-    # get_order should not be called!!
-    mocker.patch('freqtrade.exchange.Exchange.get_order', MagicMock(side_effect=ValueError))
+    # fetch_order should not be called!!
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order', MagicMock(side_effect=ValueError))
     wallet_mock = MagicMock()
     mocker.patch('freqtrade.wallets.Wallets.update', wallet_mock)
 
@@ -1772,17 +1901,24 @@ def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_orde
         open_rate=0.245441,
         fee_open=0.0025,
         fee_close=0.0025,
+        open_date=arrow.utcnow().datetime,
         open_order_id="123456",
         is_open=True,
     )
-    freqtrade.update_trade_state(trade, limit_sell_order)
+    order = Order.parse_from_ccxt_object(limit_sell_order_open, 'LTC/ETH', 'sell')
+    trade.orders.append(order)
+    assert order.status == 'open'
+    freqtrade.update_trade_state(trade, trade.open_order_id, limit_sell_order)
     assert trade.amount == limit_sell_order['amount']
     # Wallet needs to be updated after closing a limit-sell order to reenable buying
     assert wallet_mock.call_count == 1
     assert not trade.is_open
+    # Order is updated by update_trade_state
+    assert order.status == 'closed'
 
 
-def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mocker) -> None:
+def test_handle_trade(default_conf, limit_buy_order, limit_sell_order_open, limit_sell_order,
+                      fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
@@ -1792,8 +1928,8 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock
             'ask': 0.00001173,
             'last': 0.00001172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
-        sell=MagicMock(return_value={'id': limit_sell_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order),
+        sell=MagicMock(return_value=limit_sell_order_open),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -1822,13 +1958,14 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock
     assert trade.close_date is not None
 
 
-def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order, fee, mocker) -> None:
+def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order_open,
+                                    fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
 
@@ -1873,7 +2010,7 @@ def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order, fee,
     assert freqtrade.handle_trade(trades[0]) is True
 
 
-def test_handle_trade_roi(default_conf, ticker, limit_buy_order,
+def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open,
                           fee, mocker, caplog) -> None:
     caplog.set_level(logging.DEBUG)
 
@@ -1881,7 +2018,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order,
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
 
@@ -1906,14 +2043,14 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order,
 
 
 def test_handle_trade_use_sell_signal(
-        default_conf, ticker, limit_buy_order, fee, mocker, caplog) -> None:
+        default_conf, ticker, limit_buy_order_open, fee, mocker, caplog) -> None:
     # use_sell_signal is True buy default
     caplog.set_level(logging.DEBUG)
     patch_RPCManager(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
 
@@ -1934,14 +2071,14 @@ def test_handle_trade_use_sell_signal(
                    caplog)
 
 
-def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order,
+def test_close_trade(default_conf, ticker, limit_buy_order, limit_buy_order_open, limit_sell_order,
                      fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -1961,17 +2098,34 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order,
         freqtrade.handle_trade(trade)
 
 
+def test_bot_loop_start_called_once(mocker, default_conf, caplog):
+    ftbot = get_patched_freqtradebot(mocker, default_conf)
+    patch_get_signal(ftbot)
+    ftbot.strategy.bot_loop_start = MagicMock(side_effect=ValueError)
+    ftbot.strategy.analyze = MagicMock()
+
+    ftbot.process()
+    assert log_has_re(r'Strategy caused the following exception.*', caplog)
+    assert ftbot.strategy.bot_loop_start.call_count == 1
+    assert ftbot.strategy.analyze.call_count == 1
+
+
 def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_order_old, open_trade,
                                               fee, mocker) -> None:
     default_conf["unfilledtimeout"] = {"buy": 1400, "sell": 30}
 
     rpc_mock = patch_RPCManager(mocker)
     cancel_order_mock = MagicMock(return_value=limit_buy_order_old)
+    cancel_buy_order = deepcopy(limit_buy_order_old)
+    cancel_buy_order['status'] = 'canceled'
+    cancel_order_wr_mock = MagicMock(return_value=cancel_buy_order)
+
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(return_value=limit_buy_order_old),
+        fetch_order=MagicMock(return_value=limit_buy_order_old),
+        cancel_order_with_result=cancel_order_wr_mock,
         cancel_order=cancel_order_mock,
         get_fee=fee
     )
@@ -2004,7 +2158,7 @@ def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_or
     freqtrade.strategy.check_buy_timeout = MagicMock(return_value=True)
     # Trade should be closed since the function returns true
     freqtrade.check_handle_timedout()
-    assert cancel_order_mock.call_count == 1
+    assert cancel_order_wr_mock.call_count == 1
     assert rpc_mock.call_count == 1
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     nb_trades = len(trades)
@@ -2015,12 +2169,14 @@ def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_or
 def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, open_trade,
                                    fee, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
-    cancel_order_mock = MagicMock(return_value=limit_buy_order_old)
+    limit_buy_cancel = deepcopy(limit_buy_order_old)
+    limit_buy_cancel['status'] = 'canceled'
+    cancel_order_mock = MagicMock(return_value=limit_buy_cancel)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(return_value=limit_buy_order_old),
+        fetch_order=MagicMock(return_value=limit_buy_order_old),
         cancel_order_with_result=cancel_order_mock,
         get_fee=fee
     )
@@ -2050,7 +2206,7 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(return_value=limit_buy_order_old),
+        fetch_order=MagicMock(return_value=limit_buy_order_old),
         cancel_order=cancel_order_mock,
         get_fee=fee
     )
@@ -2077,7 +2233,7 @@ def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_ord
         'freqtrade.exchange.Exchange',
         validate_pairs=MagicMock(),
         fetch_ticker=ticker,
-        get_order=MagicMock(side_effect=DependencyException),
+        fetch_order=MagicMock(side_effect=ExchangeError),
         cancel_order=cancel_order_mock,
         get_fee=fee
     )
@@ -2103,7 +2259,7 @@ def test_check_handle_timedout_sell_usercustom(default_conf, ticker, limit_sell_
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(return_value=limit_sell_order_old),
+        fetch_order=MagicMock(return_value=limit_sell_order_old),
         cancel_order=cancel_order_mock
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -2150,7 +2306,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old,
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(return_value=limit_sell_order_old),
+        fetch_order=MagicMock(return_value=limit_sell_order_old),
         cancel_order=cancel_order_mock
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -2181,7 +2337,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old,
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(return_value=limit_sell_order_old),
+        fetch_order=MagicMock(return_value=limit_sell_order_old),
         cancel_order_with_result=cancel_order_mock
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -2203,12 +2359,15 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old,
 def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial,
                                        open_trade, mocker) -> None:
     rpc_mock = patch_RPCManager(mocker)
-    cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial)
+    limit_buy_canceled = deepcopy(limit_buy_order_old_partial)
+    limit_buy_canceled['status'] = 'canceled'
+
+    cancel_order_mock = MagicMock(return_value=limit_buy_canceled)
     patch_exchange(mocker)
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(return_value=limit_buy_order_old_partial),
+        fetch_order=MagicMock(return_value=limit_buy_order_old_partial),
         cancel_order_with_result=cancel_order_mock
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -2219,7 +2378,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old
     # note this is for a partially-complete buy order
     freqtrade.check_handle_timedout()
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 1
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     assert trades[0].amount == 23.0
@@ -2236,7 +2395,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(return_value=limit_buy_order_old_partial),
+        fetch_order=MagicMock(return_value=limit_buy_order_old_partial),
         cancel_order_with_result=cancel_order_mock,
         get_trades_for_order=MagicMock(return_value=trades_for_order),
     )
@@ -2254,7 +2413,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap
     assert log_has_re(r"Applying fee on amount for Trade.*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 1
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -2274,7 +2433,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade,
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(return_value=limit_buy_order_old_partial),
+        fetch_order=MagicMock(return_value=limit_buy_order_old_partial),
         cancel_order_with_result=cancel_order_mock,
         get_trades_for_order=MagicMock(return_value=trades_for_order),
     )
@@ -2294,7 +2453,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade,
     assert log_has_re(r"Could not update trade amount: .*", caplog)
 
     assert cancel_order_mock.call_count == 1
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 1
     trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all()
     assert len(trades) == 1
     # Verify that trade has been updated
@@ -2318,7 +2477,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')),
+        fetch_order=MagicMock(side_effect=ExchangeError('Oh snap')),
         cancel_order=cancel_order_mock
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -2336,7 +2495,11 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke
 def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
-    cancel_order_mock = MagicMock(return_value=limit_buy_order)
+    cancel_buy_order = deepcopy(limit_buy_order)
+    cancel_buy_order['status'] = 'canceled'
+    del cancel_buy_order['filled']
+
+    cancel_order_mock = MagicMock(return_value=cancel_buy_order)
     mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock)
 
     freqtrade = FreqtradeBot(default_conf)
@@ -2356,9 +2519,12 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non
     assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
     assert cancel_order_mock.call_count == 1
 
-    limit_buy_order['filled'] = 2
-    mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException)
+    # Order remained open for some reason (cancel failed)
+    cancel_buy_order['status'] = 'open'
+    cancel_order_mock = MagicMock(return_value=cancel_buy_order)
+    mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock)
     assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason)
+    assert log_has_re(r"Order .* for .* not cancelled.", caplog)
 
 
 @pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'],
@@ -2450,13 +2616,15 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None:
     send_msg_mock.reset_mock()
 
     order['amount'] = 2
-    assert freqtrade.handle_cancel_sell(trade, order, reason) == CANCEL_REASON['PARTIALLY_FILLED']
+    assert freqtrade.handle_cancel_sell(trade, order, reason
+                                        ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
     # Assert cancel_order was not called (callcount remains unchanged)
     assert cancel_order_mock.call_count == 1
     assert send_msg_mock.call_count == 1
-    assert freqtrade.handle_cancel_sell(trade, order, reason) == CANCEL_REASON['PARTIALLY_FILLED']
+    assert freqtrade.handle_cancel_sell(trade, order, reason
+                                        ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
     # Message should not be iterated again
-    assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED']
+    assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN']
     assert send_msg_mock.call_count == 1
 
 
@@ -2487,30 +2655,42 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N
     patch_whitelist(mocker, default_conf)
     freqtrade = FreqtradeBot(default_conf)
     patch_get_signal(freqtrade)
+    freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=False)
 
     # Create some test data
     freqtrade.enter_positions()
+    rpc_mock.reset_mock()
 
     trade = Trade.query.first()
     assert trade
+    assert freqtrade.strategy.confirm_trade_exit.call_count == 0
 
     # Increase the price and sell it
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker_sell_up
     )
+    # Prevented sell ...
+    freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI)
+    assert rpc_mock.call_count == 0
+    assert freqtrade.strategy.confirm_trade_exit.call_count == 1
+
+    # Repatch with true
+    freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=True)
 
     freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI)
+    assert freqtrade.strategy.confirm_trade_exit.call_count == 1
 
-    assert rpc_mock.call_count == 2
+    assert rpc_mock.call_count == 1
     last_msg = rpc_mock.call_args_list[-1][0][0]
     assert {
+        'trade_id': 1,
         'type': RPCMessageType.SELL_NOTIFICATION,
         'exchange': 'Bittrex',
         'pair': 'ETH/BTC',
         'gain': 'profit',
         'limit': 1.172e-05,
-        'amount': 91.07468123861567,
+        'amount': 91.07468123,
         'order_type': 'limit',
         'open_rate': 1.098e-05,
         'current_rate': 1.173e-05,
@@ -2555,11 +2735,12 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker)
     last_msg = rpc_mock.call_args_list[-1][0][0]
     assert {
         'type': RPCMessageType.SELL_NOTIFICATION,
+        'trade_id': 1,
         'exchange': 'Bittrex',
         'pair': 'ETH/BTC',
         'gain': 'loss',
         'limit': 1.044e-05,
-        'amount': 91.07468123861567,
+        'amount': 91.07468123,
         'order_type': 'limit',
         'open_rate': 1.098e-05,
         'current_rate': 1.043e-05,
@@ -2611,11 +2792,12 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe
 
     assert {
         'type': RPCMessageType.SELL_NOTIFICATION,
+        'trade_id': 1,
         'exchange': 'Bittrex',
         'pair': 'ETH/BTC',
         'gain': 'loss',
         'limit': 1.08801e-05,
-        'amount': 91.07468123861567,
+        'amount': 91.07468123,
         'order_type': 'limit',
         'open_rate': 1.098e-05,
         'current_rate': 1.043e-05,
@@ -2632,7 +2814,8 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe
 
 def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, caplog) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
-    mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException())
+    mocker.patch('freqtrade.exchange.Exchange.cancel_stoploss_order',
+                 side_effect=InvalidOrderException())
     mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=300))
     sellmock = MagicMock()
     patch_exchange(mocker)
@@ -2649,6 +2832,7 @@ def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, c
 
     trade = Trade.query.first()
     Trade.session = MagicMock()
+    PairLock.session = MagicMock()
 
     freqtrade.config['dry_run'] = False
     trade.stoploss_order_id = "abcd"
@@ -2680,7 +2864,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke
         amount_to_precision=lambda s, x, y: y,
         price_to_precision=lambda s, x, y: y,
         stoploss=stoploss,
-        cancel_order=cancel_order,
+        cancel_stoploss_order=cancel_order,
     )
 
     freqtrade = FreqtradeBot(default_conf)
@@ -2713,7 +2897,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke
 
 
 def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, fee,
-                                                         limit_buy_order, mocker) -> None:
+                                                         mocker) -> None:
     default_conf['exchange']['name'] = 'binance'
     rpc_mock = patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -2771,7 +2955,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f
         "fee": None,
         "trades": None
     })
-    mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_executed)
+    mocker.patch('freqtrade.exchange.Exchange.fetch_stoploss_order', stoploss_executed)
 
     freqtrade.exit_positions(trades)
     assert trade.stoploss_order_id is None
@@ -2815,11 +2999,12 @@ def test_execute_sell_market_order(default_conf, ticker, fee,
     last_msg = rpc_mock.call_args_list[-1][0][0]
     assert {
         'type': RPCMessageType.SELL_NOTIFICATION,
+        'trade_id': 1,
         'exchange': 'Bittrex',
         'pair': 'ETH/BTC',
         'gain': 'profit',
         'limit': 1.172e-05,
-        'amount': 91.07468123861567,
+        'amount': 91.07468123,
         'order_type': 'market',
         'open_rate': 1.098e-05,
         'current_rate': 1.173e-05,
@@ -2834,7 +3019,36 @@ def test_execute_sell_market_order(default_conf, ticker, fee,
     } == last_msg
 
 
-def test_sell_profit_only_enable_profit(default_conf, limit_buy_order,
+def test_execute_sell_insufficient_funds_error(default_conf, ticker, fee,
+                                               ticker_sell_up, mocker) -> None:
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mock_insuf = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_insufficient_funds')
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        fetch_ticker=ticker,
+        get_fee=fee,
+        sell=MagicMock(side_effect=InsufficientFundsError())
+    )
+    patch_get_signal(freqtrade)
+
+    # Create some test data
+    freqtrade.enter_positions()
+
+    trade = Trade.query.first()
+    assert trade
+
+    # Increase the price and sell it
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        fetch_ticker=ticker_sell_up
+    )
+
+    assert not freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'],
+                                      sell_reason=SellType.ROI)
+    assert mock_insuf.call_count == 1
+
+
+def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, limit_buy_order_open,
                                         fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -2845,7 +3059,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order,
             'ask': 0.00002173,
             'last': 0.00002172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     default_conf['ask_strategy'] = {
@@ -2866,7 +3080,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order,
     assert trade.sell_reason == SellType.SELL_SIGNAL.value
 
 
-def test_sell_profit_only_disable_profit(default_conf, limit_buy_order,
+def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, limit_buy_order_open,
                                          fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -2877,7 +3091,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order,
             'ask': 0.00002173,
             'last': 0.00002172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     default_conf['ask_strategy'] = {
@@ -2897,7 +3111,8 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order,
     assert trade.sell_reason == SellType.SELL_SIGNAL.value
 
 
-def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker) -> None:
+def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, limit_buy_order_open,
+                                      fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
@@ -2907,7 +3122,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker
             'ask': 0.00000173,
             'last': 0.00000172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     default_conf['ask_strategy'] = {
@@ -2926,7 +3141,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker
     assert freqtrade.handle_trade(trade) is False
 
 
-def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocker) -> None:
+def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, limit_buy_order_open,
+                                       fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
@@ -2936,7 +3152,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke
             'ask': 0.0000173,
             'last': 0.0000172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     default_conf['ask_strategy'] = {
@@ -2958,7 +3174,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke
     assert trade.sell_reason == SellType.SELL_SIGNAL.value
 
 
-def test_sell_not_enough_balance(default_conf, limit_buy_order,
+def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_open,
                                  fee, mocker, caplog) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -2969,7 +3185,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order,
             'ask': 0.00002173,
             'last': 0.00002172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
 
@@ -3067,17 +3283,17 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo
     freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'],
                            sell_reason=SellType.STOP_LOSS)
     trade.close(ticker_sell_down()['bid'])
-    assert trade.pair in freqtrade.strategy._pair_locked_until
     assert freqtrade.strategy.is_pair_locked(trade.pair)
 
     # reinit - should buy other pair.
     caplog.clear()
     freqtrade.enter_positions()
 
-    assert log_has(f"Pair {trade.pair} is currently locked.", caplog)
+    assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog)
 
 
-def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> None:
+def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open,
+                                  fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
@@ -3087,7 +3303,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) ->
             'ask': 0.0000173,
             'last': 0.0000172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     default_conf['ask_strategy'] = {
@@ -3111,7 +3327,8 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) ->
     assert trade.sell_reason == SellType.ROI.value
 
 
-def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) -> None:
+def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order,
+                            fee, caplog, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
     mocker.patch.multiple(
@@ -3121,7 +3338,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker)
             'ask': 0.00001099,
             'last': 0.00001099
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     default_conf['trailing_stop'] = True
@@ -3161,7 +3378,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker)
     assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value
 
 
-def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee,
+def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_order_open, fee,
                                      caplog, mocker) -> None:
     buy_price = limit_buy_order['price']
     patch_RPCManager(mocker)
@@ -3173,7 +3390,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee,
             'ask': buy_price - 0.000001,
             'last': buy_price - 0.000001
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     default_conf['trailing_stop'] = True
@@ -3218,7 +3435,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee,
         f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog)
 
 
-def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee,
+def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_order_open, fee,
                                    caplog, mocker) -> None:
     buy_price = limit_buy_order['price']
     patch_RPCManager(mocker)
@@ -3230,7 +3447,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee,
             'ask': buy_price - 0.000001,
             'last': buy_price - 0.000001
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     patch_whitelist(mocker, default_conf)
@@ -3276,7 +3493,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee,
     assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value
 
 
-def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee,
+def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_open, fee,
                                  caplog, mocker) -> None:
     buy_price = limit_buy_order['price']
     # buy_price: 0.00001099
@@ -3290,7 +3507,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee,
             'ask': buy_price,
             'last': buy_price
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     patch_whitelist(mocker, default_conf)
@@ -3339,7 +3556,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee,
     assert trade.stop_loss == 0.0000117705
 
 
-def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order,
+def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open,
                                           fee, mocker) -> None:
     patch_RPCManager(mocker)
     patch_exchange(mocker)
@@ -3350,7 +3567,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order,
             'ask': 0.00000173,
             'last': 0.00000172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
     default_conf['ask_strategy'] = {
@@ -3371,7 +3588,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order,
     # Test if buy-signal is absent
     patch_get_signal(freqtrade, value=(False, True))
     assert freqtrade.handle_trade(trade) is True
-    assert trade.sell_reason == SellType.STOP_LOSS.value
+    assert trade.sell_reason == SellType.SELL_SIGNAL.value
 
 
 def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker):
@@ -3527,6 +3744,48 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c
                    'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).',
                    caplog)
 
+    assert trade.fee_open == 0.001
+    assert trade.fee_close == 0.001
+    assert trade.fee_open_cost is not None
+    assert trade.fee_open_currency is not None
+    assert trade.fee_close_cost is None
+    assert trade.fee_close_currency is None
+
+
+def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee,
+                                mocker, markets):
+    # Different fee currency on both trades
+    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order3)
+    amount = float(sum(x['amount'] for x in trades_for_order3))
+    default_conf['stake_currency'] = 'ETH'
+    trade = Trade(
+        pair='LTC/ETH',
+        amount=amount,
+        exchange='binance',
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.245441,
+        open_order_id="123456"
+    )
+    # Fake markets entry to enable fee parsing
+    markets['BNB/ETH'] = markets['ETH/BTC']
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
+    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
+                 return_value={'ask': 0.19, 'last': 0.2})
+
+    # Amount is reduced by "fee"
+    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.0005)
+    assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
+                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).',
+                   caplog)
+    # Overall fee is average of both trade's fee
+    assert trade.fee_open == 0.001518575
+    assert trade.fee_open_cost is not None
+    assert trade.fee_open_currency is not None
+    assert trade.fee_close_cost is None
+    assert trade.fee_close_currency is None
+
 
 def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee,
                                    caplog, mocker):
@@ -3692,8 +3951,8 @@ def test_apply_fee_conditional(default_conf, fee, caplog, mocker,
     assert walletmock.call_count == 1
 
 
-def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker,
-                                    order_book_l2):
+def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order_open, limit_buy_order,
+                                    fee, mocker, order_book_l2):
     default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True
     default_conf['bid_strategy']['check_depth_of_market']['bids_to_ask_delta'] = 0.1
     patch_RPCManager(mocker)
@@ -3702,7 +3961,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee,
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
 
@@ -3817,8 +4076,8 @@ def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None:
     assert freqtrade._check_depth_of_market_buy('ETH/BTC', conf) is False
 
 
-def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order,
-                                 fee, mocker, order_book_l2, caplog) -> None:
+def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_order, fee,
+                                 limit_sell_order_open, mocker, order_book_l2, caplog) -> None:
     """
     test order book ask strategy
     """
@@ -3837,8 +4096,8 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order
             'ask': 0.00001173,
             'last': 0.00001172
         }),
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
-        sell=MagicMock(return_value={'id': limit_sell_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
+        sell=MagicMock(return_value=limit_sell_order_open),
         get_fee=fee,
     )
     freqtrade = FreqtradeBot(default_conf)
@@ -3877,6 +4136,8 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order
     ('ask', 0.006, 1.0, 0.006),
 ])
 def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, expected) -> None:
+    caplog.set_level(logging.DEBUG)
+
     default_conf['ask_strategy']['price_side'] = side
     mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'ask': ask, 'bid': bid})
     pair = "ETH/BTC"
@@ -3898,6 +4159,7 @@ def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, expected) -
     ('ask', 0.043949),  # Value from order_book_l2 fiture - asks side
 ])
 def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2):
+    caplog.set_level(logging.DEBUG)
     # Test orderbook mode
     default_conf['ask_strategy']['price_side'] = side
     default_conf['ask_strategy']['use_order_book'] = True
@@ -3980,7 +4242,7 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker):
 
 
 @pytest.mark.usefixtures("init_persistence")
-def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, caplog):
+def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_open, caplog):
     default_conf['dry_run'] = True
     # Initialize to 2 times stake amount
     default_conf['dry_run_wallet'] = 0.002
@@ -3990,7 +4252,7 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order,
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
         fetch_ticker=ticker,
-        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        buy=MagicMock(return_value=limit_buy_order_open),
         get_fee=fee,
     )
 
@@ -4014,15 +4276,275 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order,
 @pytest.mark.usefixtures("init_persistence")
 def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limit_sell_order):
     default_conf['cancel_open_orders_on_exit'] = True
-    mocker.patch('freqtrade.exchange.Exchange.get_order',
-                 side_effect=[DependencyException(), limit_sell_order, limit_buy_order])
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order',
+                 side_effect=[
+                     ExchangeError(), limit_sell_order, limit_buy_order, limit_sell_order])
     buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_buy')
     sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_sell')
 
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
     create_mock_trades(fee)
     trades = Trade.query.all()
-    assert len(trades) == 3
+    assert len(trades) == MOCK_TRADE_COUNT
     freqtrade.cancel_all_open_orders()
     assert buy_mock.call_count == 1
-    assert sell_mock.call_count == 1
+    assert sell_mock.call_count == 2
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_check_for_open_trades(mocker, default_conf, fee):
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+
+    freqtrade.check_for_open_trades()
+    assert freqtrade.rpc.send_msg.call_count == 0
+
+    create_mock_trades(fee)
+    trade = Trade.query.first()
+    trade.is_open = True
+
+    freqtrade.check_for_open_trades()
+    assert freqtrade.rpc.send_msg.call_count == 1
+    assert 'Handle these trades manually' in freqtrade.rpc.send_msg.call_args[0][0]['status']
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_update_open_orders(mocker, default_conf, fee, caplog):
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    create_mock_trades(fee)
+
+    freqtrade.update_open_orders()
+    assert log_has_re(r"Error updating Order .*", caplog)
+    caplog.clear()
+
+    assert len(Order.get_open_orders()) == 3
+    matching_buy_order = mock_order_4()
+    matching_buy_order.update({
+        'status': 'closed',
+    })
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order', return_value=matching_buy_order)
+    freqtrade.update_open_orders()
+    # Only stoploss and sell orders are kept open
+    assert len(Order.get_open_orders()) == 2
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee):
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+
+    def patch_with_fee(order):
+        order.update({'fee': {'cost': 0.1, 'rate': 0.01,
+                      'currency': order['symbol'].split('/')[0]}})
+        return order
+
+    mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order',
+                 side_effect=[
+                     patch_with_fee(mock_order_2_sell()),
+                     patch_with_fee(mock_order_3_sell()),
+                     patch_with_fee(mock_order_1()),
+                     patch_with_fee(mock_order_2()),
+                     patch_with_fee(mock_order_3()),
+                     patch_with_fee(mock_order_4()),
+                 ]
+                 )
+
+    create_mock_trades(fee)
+    trades = Trade.get_trades().all()
+    assert len(trades) == MOCK_TRADE_COUNT
+    for trade in trades:
+        assert trade.fee_open_cost is None
+        assert trade.fee_open_currency is None
+        assert trade.fee_close_cost is None
+        assert trade.fee_close_currency is None
+
+    freqtrade.update_closed_trades_without_assigned_fees()
+
+    trades = Trade.get_trades().all()
+    assert len(trades) == MOCK_TRADE_COUNT
+
+    for trade in trades:
+        if trade.is_open:
+            # Exclude Trade 4 - as the order is still open.
+            if trade.select_order('buy', False):
+                assert trade.fee_open_cost is not None
+                assert trade.fee_open_currency is not None
+            else:
+                assert trade.fee_open_cost is None
+                assert trade.fee_open_currency is None
+
+        else:
+            assert trade.fee_close_cost is not None
+            assert trade.fee_close_currency is not None
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog):
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state')
+
+    create_mock_trades(fee)
+    trades = Trade.get_trades().all()
+
+    freqtrade.reupdate_buy_order_fees(trades[0])
+    assert log_has_re(r"Trying to reupdate buy fees for .*", caplog)
+    assert mock_uts.call_count == 1
+    assert mock_uts.call_args_list[0][0][0] == trades[0]
+    assert mock_uts.call_args_list[0][0][1] == mock_order_1()['id']
+    assert log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog)
+    mock_uts.reset_mock()
+    caplog.clear()
+
+    # Test with trade without orders
+    trade = Trade(
+        pair='XRP/ETH',
+        stake_amount=0.001,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_date=arrow.utcnow().datetime,
+        is_open=True,
+        amount=20,
+        open_rate=0.01,
+        exchange='bittrex',
+    )
+    Trade.session.add(trade)
+
+    freqtrade.reupdate_buy_order_fees(trade)
+    assert log_has_re(r"Trying to reupdate buy fees for .*", caplog)
+    assert mock_uts.call_count == 0
+    assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog)
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_handle_insufficient_funds(mocker, default_conf, fee):
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mock_rlo = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.refind_lost_order')
+    mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_buy_order_fees')
+    create_mock_trades(fee)
+    trades = Trade.get_trades().all()
+
+    # Trade 0 has only a open buy order, no closed order
+    freqtrade.handle_insufficient_funds(trades[0])
+    assert mock_rlo.call_count == 0
+    assert mock_bof.call_count == 1
+
+    mock_rlo.reset_mock()
+    mock_bof.reset_mock()
+
+    # Trade 1 has closed buy and sell orders
+    freqtrade.handle_insufficient_funds(trades[1])
+    assert mock_rlo.call_count == 1
+    assert mock_bof.call_count == 0
+
+    mock_rlo.reset_mock()
+    mock_bof.reset_mock()
+
+    # Trade 2 has closed buy and sell orders
+    freqtrade.handle_insufficient_funds(trades[2])
+    assert mock_rlo.call_count == 1
+    assert mock_bof.call_count == 0
+
+    mock_rlo.reset_mock()
+    mock_bof.reset_mock()
+
+    # Trade 3 has an opne buy order
+    freqtrade.handle_insufficient_funds(trades[3])
+    assert mock_rlo.call_count == 0
+    assert mock_bof.call_count == 1
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_refind_lost_order(mocker, default_conf, fee, caplog):
+    caplog.set_level(logging.DEBUG)
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state')
+
+    mock_fo = mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order',
+                           return_value={'status': 'open'})
+
+    def reset_open_orders(trade):
+        trade.open_order_id = None
+        trade.stoploss_order_id = None
+
+    create_mock_trades(fee)
+    trades = Trade.get_trades().all()
+
+    caplog.clear()
+
+    # No open order
+    trade = trades[0]
+    reset_open_orders(trade)
+    assert trade.open_order_id is None
+    assert trade.stoploss_order_id is None
+
+    freqtrade.refind_lost_order(trade)
+    order = mock_order_1()
+    assert log_has_re(r"Order Order(.*order_id=" + order['id'] + ".*) is no longer open.", caplog)
+    assert mock_fo.call_count == 0
+    assert mock_uts.call_count == 0
+    # No change to orderid - as update_trade_state is mocked
+    assert trade.open_order_id is None
+    assert trade.stoploss_order_id is None
+
+    caplog.clear()
+    mock_fo.reset_mock()
+
+    # Open buy order
+    trade = trades[3]
+    reset_open_orders(trade)
+    assert trade.open_order_id is None
+    assert trade.stoploss_order_id is None
+
+    freqtrade.refind_lost_order(trade)
+    order = mock_order_4()
+    assert log_has_re(r"Trying to refind Order\(.*", caplog)
+    assert mock_fo.call_count == 0
+    assert mock_uts.call_count == 0
+    # No change to orderid - as update_trade_state is mocked
+    assert trade.open_order_id is None
+    assert trade.stoploss_order_id is None
+
+    caplog.clear()
+    mock_fo.reset_mock()
+
+    # Open stoploss order
+    trade = trades[4]
+    reset_open_orders(trade)
+    assert trade.open_order_id is None
+    assert trade.stoploss_order_id is None
+
+    freqtrade.refind_lost_order(trade)
+    order = mock_order_5_stoploss()
+    assert log_has_re(r"Trying to refind Order\(.*", caplog)
+    assert mock_fo.call_count == 1
+    assert mock_uts.call_count == 1
+    # stoploss_order_id is "refound" and added to the trade
+    assert trade.open_order_id is None
+    assert trade.stoploss_order_id is not None
+
+    caplog.clear()
+    mock_fo.reset_mock()
+    mock_uts.reset_mock()
+
+    # Open sell order
+    trade = trades[5]
+    reset_open_orders(trade)
+    assert trade.open_order_id is None
+    assert trade.stoploss_order_id is None
+
+    freqtrade.refind_lost_order(trade)
+    order = mock_order_6_sell()
+    assert log_has_re(r"Trying to refind Order\(.*", caplog)
+    assert mock_fo.call_count == 1
+    assert mock_uts.call_count == 1
+    # sell-orderid is "refound" and added to the trade
+    assert trade.open_order_id == order['id']
+    assert trade.stoploss_order_id is None
+
+    caplog.clear()
+
+    # Test error case
+    mock_fo = mocker.patch('freqtrade.exchange.Exchange.fetch_order_or_stoploss_order',
+                           side_effect=ExchangeError())
+    order = mock_order_5_stoploss()
+
+    freqtrade.refind_lost_order(trades[4])
+    assert log_has(f"Error updating {order['id']}.", caplog)
diff --git a/tests/test_indicators.py b/tests/test_indicators.py
new file mode 100644
index 000000000..8d02330a1
--- /dev/null
+++ b/tests/test_indicators.py
@@ -0,0 +1,19 @@
+import numpy as np
+import pandas as pd
+
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+
+def test_crossed_numpy_types():
+    """
+    This test is only present since this method currently diverges from the qtpylib implementation.
+    And we must ensure to not break this again once we update from the original source.
+    """
+    series = pd.Series([56, 97, 19, 76, 65, 25, 87, 91, 79, 79])
+    expected_result = pd.Series([False, True, False, True, False, False, True, False, False, False])
+
+    assert qtpylib.crossed_above(series, 60).equals(expected_result)
+    assert qtpylib.crossed_above(series, 60.0).equals(expected_result)
+    assert qtpylib.crossed_above(series, np.int32(60)).equals(expected_result)
+    assert qtpylib.crossed_above(series, np.int64(60)).equals(expected_result)
+    assert qtpylib.crossed_above(series, np.float64(60.0)).equals(expected_result)
diff --git a/tests/test_integration.py b/tests/test_integration.py
index 1396e86f5..9695977ac 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -62,8 +62,8 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
         get_fee=fee,
         amount_to_precision=lambda s, x, y: y,
         price_to_precision=lambda s, x, y: y,
-        get_order=stoploss_order_mock,
-        cancel_order=cancel_order_mock,
+        fetch_stoploss_order=stoploss_order_mock,
+        cancel_stoploss_order=cancel_order_mock,
     )
 
     mocker.patch.multiple(
@@ -79,10 +79,15 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
     freqtrade.strategy.order_types['stoploss_on_exchange'] = True
     # Switch ordertype to market to close trade immediately
     freqtrade.strategy.order_types['sell'] = 'market'
+    freqtrade.strategy.confirm_trade_entry = MagicMock(return_value=True)
+    freqtrade.strategy.confirm_trade_exit = MagicMock(return_value=True)
     patch_get_signal(freqtrade)
 
     # Create some test data
     freqtrade.enter_positions()
+    assert freqtrade.strategy.confirm_trade_entry.call_count == 3
+    freqtrade.strategy.confirm_trade_entry.reset_mock()
+    assert freqtrade.strategy.confirm_trade_exit.call_count == 0
     wallets_mock.reset_mock()
     Trade.session = MagicMock()
 
@@ -95,6 +100,9 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee,
     n = freqtrade.exit_positions(trades)
     assert n == 2
     assert should_sell_mock.call_count == 2
+    assert freqtrade.strategy.confirm_trade_entry.call_count == 0
+    assert freqtrade.strategy.confirm_trade_exit.call_count == 1
+    freqtrade.strategy.confirm_trade_exit.reset_mock()
 
     # Only order for 3rd trade needs to be cancelled
     assert cancel_order_mock.call_count == 1
diff --git a/tests/test_main.py b/tests/test_main.py
index 11d0ede3a..f55aea336 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -1,11 +1,11 @@
 # pragma pylint: disable=missing-docstring
 
 from copy import deepcopy
+from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
 
 import pytest
 
-from pathlib import Path
 from freqtrade.commands import Arguments
 from freqtrade.exceptions import FreqtradeException, OperationalException
 from freqtrade.freqtradebot import FreqtradeBot
@@ -35,28 +35,28 @@ def test_parse_args_backtesting(mocker) -> None:
         main(['backtesting'])
     assert backtesting_mock.call_count == 1
     call_args = backtesting_mock.call_args[0][0]
-    assert call_args["config"] == ['config.json']
-    assert call_args["verbosity"] == 0
-    assert call_args["command"] == 'backtesting'
-    assert call_args["func"] is not None
-    assert callable(call_args["func"])
-    assert call_args["ticker_interval"] is None
+    assert call_args['config'] == ['config.json']
+    assert call_args['verbosity'] == 0
+    assert call_args['command'] == 'backtesting'
+    assert call_args['func'] is not None
+    assert callable(call_args['func'])
+    assert call_args['timeframe'] is None
 
 
 def test_main_start_hyperopt(mocker) -> None:
-    mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True]))
+    mocker.patch.object(Path, 'is_file', MagicMock(side_effect=[False, True]))
     hyperopt_mock = mocker.patch('freqtrade.commands.start_hyperopt', MagicMock())
-    hyperopt_mock.__name__ = PropertyMock("start_hyperopt")
+    hyperopt_mock.__name__ = PropertyMock('start_hyperopt')
     # it's sys.exit(0) at the end of hyperopt
     with pytest.raises(SystemExit):
         main(['hyperopt'])
     assert hyperopt_mock.call_count == 1
     call_args = hyperopt_mock.call_args[0][0]
-    assert call_args["config"] == ['config.json']
-    assert call_args["verbosity"] == 0
-    assert call_args["command"] == 'hyperopt'
-    assert call_args["func"] is not None
-    assert callable(call_args["func"])
+    assert call_args['config'] == ['config.json']
+    assert call_args['verbosity'] == 0
+    assert call_args['command'] == 'hyperopt'
+    assert call_args['func'] is not None
+    assert callable(call_args['func'])
 
 
 def test_main_fatal_exception(mocker, default_conf, caplog) -> None:
@@ -65,7 +65,7 @@ def test_main_fatal_exception(mocker, default_conf, caplog) -> None:
     mocker.patch('freqtrade.worker.Worker._worker', MagicMock(side_effect=Exception))
     patched_configuration_load_config_file(mocker, default_conf)
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
-    mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
+    mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
 
     args = ['trade', '-c', 'config.json.example']
 
@@ -83,7 +83,7 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
     mocker.patch('freqtrade.wallets.Wallets.update', MagicMock())
-    mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
+    mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
 
     args = ['trade', '-c', 'config.json.example']
 
@@ -104,7 +104,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None:
     patched_configuration_load_config_file(mocker, default_conf)
     mocker.patch('freqtrade.wallets.Wallets.update', MagicMock())
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
-    mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
+    mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
 
     args = ['trade', '-c', 'config.json.example']
 
@@ -141,12 +141,12 @@ def test_main_operational_exception1(mocker, default_conf, caplog) -> None:
     assert log_has_re(r'SIGINT.*', caplog)
 
 
-def test_main_reload_conf(mocker, default_conf, caplog) -> None:
+def test_main_reload_config(mocker, default_conf, caplog) -> None:
     patch_exchange(mocker)
     mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cleanup', MagicMock())
     # Simulate Running, reload, running workflow
     worker_mock = MagicMock(side_effect=[State.RUNNING,
-                                         State.RELOAD_CONF,
+                                         State.RELOAD_CONFIG,
                                          State.RUNNING,
                                          OperationalException("Oh snap!")])
     mocker.patch('freqtrade.worker.Worker._worker', worker_mock)
@@ -155,7 +155,7 @@ def test_main_reload_conf(mocker, default_conf, caplog) -> None:
     reconfigure_mock = mocker.patch('freqtrade.worker.Worker._reconfigure', MagicMock())
 
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
-    mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
+    mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
 
     args = Arguments(['trade', '-c', 'config.json.example']).get_parsed_arg()
     worker = Worker(args=args, config=default_conf)
@@ -178,7 +178,7 @@ def test_reconfigure(mocker, default_conf) -> None:
     mocker.patch('freqtrade.wallets.Wallets.update', MagicMock())
     patched_configuration_load_config_file(mocker, default_conf)
     mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock())
-    mocker.patch('freqtrade.freqtradebot.persistence.init', MagicMock())
+    mocker.patch('freqtrade.freqtradebot.init_db', MagicMock())
 
     args = Arguments(['trade', '-c', 'config.json.example']).get_parsed_arg()
     worker = Worker(args=args, config=default_conf)
diff --git a/tests/test_misc.py b/tests/test_misc.py
index 9fd6164d5..6dcd9fbe5 100644
--- a/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -7,11 +7,10 @@ from unittest.mock import MagicMock
 import pytest
 
 from freqtrade.data.converter import ohlcv_to_dataframe
-from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json,
-                            file_load_json, format_ms_time, pair_to_filename,
-                            plural, render_template,
+from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json, file_load_json,
+                            format_ms_time, pair_to_filename, plural, render_template,
                             render_template_with_fallback, safe_value_fallback,
-                            shorten_date)
+                            safe_value_fallback2, shorten_date)
 
 
 def test_shorten_date() -> None:
@@ -96,24 +95,40 @@ def test_format_ms_time() -> None:
 
 
 def test_safe_value_fallback():
+    dict1 = {'keya': None, 'keyb': 2, 'keyc': 5, 'keyd': None}
+    assert safe_value_fallback(dict1, 'keya', 'keyb') == 2
+    assert safe_value_fallback(dict1, 'keyb', 'keya') == 2
+
+    assert safe_value_fallback(dict1, 'keyb', 'keyc') == 2
+    assert safe_value_fallback(dict1, 'keya', 'keyc') == 5
+
+    assert safe_value_fallback(dict1, 'keyc', 'keyb') == 5
+
+    assert safe_value_fallback(dict1, 'keya', 'keyd') is None
+
+    assert safe_value_fallback(dict1, 'keyNo', 'keyNo') is None
+    assert safe_value_fallback(dict1, 'keyNo', 'keyNo', 55) == 55
+
+
+def test_safe_value_fallback2():
     dict1 = {'keya': None, 'keyb': 2, 'keyc': 5, 'keyd': None}
     dict2 = {'keya': 20, 'keyb': None, 'keyc': 6, 'keyd': None}
-    assert safe_value_fallback(dict1, dict2, 'keya', 'keya') == 20
-    assert safe_value_fallback(dict2, dict1, 'keya', 'keya') == 20
+    assert safe_value_fallback2(dict1, dict2, 'keya', 'keya') == 20
+    assert safe_value_fallback2(dict2, dict1, 'keya', 'keya') == 20
 
-    assert safe_value_fallback(dict1, dict2, 'keyb', 'keyb') == 2
-    assert safe_value_fallback(dict2, dict1, 'keyb', 'keyb') == 2
+    assert safe_value_fallback2(dict1, dict2, 'keyb', 'keyb') == 2
+    assert safe_value_fallback2(dict2, dict1, 'keyb', 'keyb') == 2
 
-    assert safe_value_fallback(dict1, dict2, 'keyc', 'keyc') == 5
-    assert safe_value_fallback(dict2, dict1, 'keyc', 'keyc') == 6
+    assert safe_value_fallback2(dict1, dict2, 'keyc', 'keyc') == 5
+    assert safe_value_fallback2(dict2, dict1, 'keyc', 'keyc') == 6
 
-    assert safe_value_fallback(dict1, dict2, 'keyd', 'keyd') is None
-    assert safe_value_fallback(dict2, dict1, 'keyd', 'keyd') is None
-    assert safe_value_fallback(dict2, dict1, 'keyd', 'keyd', 1234) == 1234
+    assert safe_value_fallback2(dict1, dict2, 'keyd', 'keyd') is None
+    assert safe_value_fallback2(dict2, dict1, 'keyd', 'keyd') is None
+    assert safe_value_fallback2(dict2, dict1, 'keyd', 'keyd', 1234) == 1234
 
-    assert safe_value_fallback(dict1, dict2, 'keyNo', 'keyNo') is None
-    assert safe_value_fallback(dict2, dict1, 'keyNo', 'keyNo') is None
-    assert safe_value_fallback(dict2, dict1, 'keyNo', 'keyNo', 1234) == 1234
+    assert safe_value_fallback2(dict1, dict2, 'keyNo', 'keyNo') is None
+    assert safe_value_fallback2(dict2, dict1, 'keyNo', 'keyNo') is None
+    assert safe_value_fallback2(dict2, dict1, 'keyNo', 'keyNo', 1234) == 1234
 
 
 def test_plural() -> None:
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index fe2912486..7487b2ef5 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -7,14 +7,14 @@ import pytest
 from sqlalchemy import create_engine
 
 from freqtrade import constants
-from freqtrade.exceptions import OperationalException
-from freqtrade.persistence import Trade, clean_dry_run_db, init
-from tests.conftest import log_has, create_mock_trades
+from freqtrade.exceptions import DependencyException, OperationalException
+from freqtrade.persistence import Order, Trade, clean_dry_run_db, init_db
+from tests.conftest import create_mock_trades, log_has, log_has_re
 
 
 def test_init_create_session(default_conf):
     # Check if init create a session
-    init(default_conf['db_url'], default_conf['dry_run'])
+    init_db(default_conf['db_url'], default_conf['dry_run'])
     assert hasattr(Trade, 'session')
     assert 'scoped_session' in type(Trade.session).__name__
 
@@ -22,9 +22,9 @@ def test_init_create_session(default_conf):
 def test_init_custom_db_url(default_conf, mocker):
     # Update path to a value other than default, but still in-memory
     default_conf.update({'db_url': 'sqlite:///tmp/freqtrade2_test.sqlite'})
-    create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock())
+    create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
 
-    init(default_conf['db_url'], default_conf['dry_run'])
+    init_db(default_conf['db_url'], default_conf['dry_run'])
     assert create_engine_mock.call_count == 1
     assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tmp/freqtrade2_test.sqlite'
 
@@ -33,16 +33,16 @@ def test_init_invalid_db_url(default_conf):
     # Update path to a value other than default, but still in-memory
     default_conf.update({'db_url': 'unknown:///some.url'})
     with pytest.raises(OperationalException, match=r'.*no valid database URL*'):
-        init(default_conf['db_url'], default_conf['dry_run'])
+        init_db(default_conf['db_url'], default_conf['dry_run'])
 
 
 def test_init_prod_db(default_conf, mocker):
     default_conf.update({'dry_run': False})
     default_conf.update({'db_url': constants.DEFAULT_DB_PROD_URL})
 
-    create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock())
+    create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
 
-    init(default_conf['db_url'], default_conf['dry_run'])
+    init_db(default_conf['db_url'], default_conf['dry_run'])
     assert create_engine_mock.call_count == 1
     assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.sqlite'
 
@@ -51,9 +51,9 @@ def test_init_dryrun_db(default_conf, mocker):
     default_conf.update({'dry_run': True})
     default_conf.update({'db_url': constants.DEFAULT_DB_DRYRUN_URL})
 
-    create_engine_mock = mocker.patch('freqtrade.persistence.create_engine', MagicMock())
+    create_engine_mock = mocker.patch('freqtrade.persistence.models.create_engine', MagicMock())
 
-    init(default_conf['db_url'], default_conf['dry_run'])
+    init_db(default_conf['db_url'], default_conf['dry_run'])
     assert create_engine_mock.call_count == 1
     assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.dryrun.sqlite'
 
@@ -93,6 +93,8 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog):
         stake_amount=0.001,
         open_rate=0.01,
         amount=5,
+        is_open=True,
+        open_date=arrow.utcnow().datetime,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         exchange='bittrex',
@@ -107,9 +109,9 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog):
     assert trade.open_rate == 0.00001099
     assert trade.close_profit is None
     assert trade.close_date is None
-    assert log_has("LIMIT_BUY has been fulfilled for Trade(id=2, "
-                   "pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).",
-                   caplog)
+    assert log_has_re(r"LIMIT_BUY has been fulfilled for Trade\(id=2, "
+                      r"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=.*\).",
+                      caplog)
 
     caplog.clear()
     trade.open_order_id = 'something'
@@ -118,9 +120,9 @@ def test_update_with_bittrex(limit_buy_order, limit_sell_order, fee, caplog):
     assert trade.close_rate == 0.00001173
     assert trade.close_profit == 0.06201058
     assert trade.close_date is not None
-    assert log_has("LIMIT_SELL has been fulfilled for Trade(id=2, "
-                   "pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=closed).",
-                   caplog)
+    assert log_has_re(r"LIMIT_SELL has been fulfilled for Trade\(id=2, "
+                      r"pair=ETH/BTC, amount=90.99181073, open_rate=0.00001099, open_since=.*\).",
+                      caplog)
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -131,8 +133,10 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog):
         stake_amount=0.001,
         amount=5,
         open_rate=0.01,
+        is_open=True,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
+        open_date=arrow.utcnow().datetime,
         exchange='bittrex',
     )
 
@@ -142,20 +146,21 @@ def test_update_market_order(market_buy_order, market_sell_order, fee, caplog):
     assert trade.open_rate == 0.00004099
     assert trade.close_profit is None
     assert trade.close_date is None
-    assert log_has("MARKET_BUY has been fulfilled for Trade(id=1, "
-                   "pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).",
-                   caplog)
+    assert log_has_re(r"MARKET_BUY has been fulfilled for Trade\(id=1, "
+                      r"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=.*\).",
+                      caplog)
 
     caplog.clear()
+    trade.is_open = True
     trade.open_order_id = 'something'
     trade.update(market_sell_order)
     assert trade.open_order_id is None
     assert trade.close_rate == 0.00004173
     assert trade.close_profit == 0.01297561
     assert trade.close_date is not None
-    assert log_has("MARKET_SELL has been fulfilled for Trade(id=1, "
-                   "pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=closed).",
-                   caplog)
+    assert log_has_re(r"MARKET_SELL has been fulfilled for Trade\(id=1, "
+                      r"pair=ETH/BTC, amount=91.99181073, open_rate=0.00004099, open_since=.*\).",
+                      caplog)
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -172,10 +177,10 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee):
 
     trade.open_order_id = 'something'
     trade.update(limit_buy_order)
-    assert trade._calc_open_trade_price() == 0.0010024999999225068
+    assert trade._calc_open_trade_value() == 0.0010024999999225068
 
     trade.update(limit_sell_order)
-    assert trade.calc_close_trade_price() == 0.0010646656050132426
+    assert trade.calc_close_trade_value() == 0.0010646656050132426
 
     # Profit in BTC
     assert trade.calc_profit() == 0.00006217
@@ -184,6 +189,36 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee):
     assert trade.calc_profit_ratio() == 0.06201058
 
 
+@pytest.mark.usefixtures("init_persistence")
+def test_trade_close(limit_buy_order, limit_sell_order, fee):
+    trade = Trade(
+        pair='ETH/BTC',
+        stake_amount=0.001,
+        open_rate=0.01,
+        amount=5,
+        is_open=True,
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_date=arrow.Arrow(2020, 2, 1, 15, 5, 1).datetime,
+        exchange='bittrex',
+    )
+    assert trade.close_profit is None
+    assert trade.close_date is None
+    assert trade.is_open is True
+    trade.close(0.02)
+    assert trade.is_open is False
+    assert trade.close_profit == 0.99002494
+    assert trade.close_date is not None
+
+    new_date = arrow.Arrow(2020, 2, 2, 15, 6, 1).datetime,
+    assert trade.close_date != new_date
+    # Close should NOT update close_date if the trade has been closed already
+    assert trade.is_open is False
+    trade.close_date = new_date
+    trade.close(0.02)
+    assert trade.close_date == new_date
+
+
 @pytest.mark.usefixtures("init_persistence")
 def test_calc_close_trade_price_exception(limit_buy_order, fee):
     trade = Trade(
@@ -198,7 +233,7 @@ def test_calc_close_trade_price_exception(limit_buy_order, fee):
 
     trade.open_order_id = 'something'
     trade.update(limit_buy_order)
-    assert trade.calc_close_trade_price() == 0.0
+    assert trade.calc_close_trade_value() == 0.0
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -242,7 +277,7 @@ def test_update_invalid_order(limit_buy_order):
 
 
 @pytest.mark.usefixtures("init_persistence")
-def test_calc_open_trade_price(limit_buy_order, fee):
+def test_calc_open_trade_value(limit_buy_order, fee):
     trade = Trade(
         pair='ETH/BTC',
         stake_amount=0.001,
@@ -256,10 +291,10 @@ def test_calc_open_trade_price(limit_buy_order, fee):
     trade.update(limit_buy_order)  # Buy @ 0.00001099
 
     # Get the open rate price with the standard fee rate
-    assert trade._calc_open_trade_price() == 0.0010024999999225068
+    assert trade._calc_open_trade_value() == 0.0010024999999225068
     trade.fee_open = 0.003
     # Get the open rate price with a custom fee rate
-    assert trade._calc_open_trade_price() == 0.001002999999922468
+    assert trade._calc_open_trade_value() == 0.001002999999922468
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -277,14 +312,14 @@ def test_calc_close_trade_price(limit_buy_order, limit_sell_order, fee):
     trade.update(limit_buy_order)  # Buy @ 0.00001099
 
     # Get the close rate price with a custom close rate and a regular fee rate
-    assert trade.calc_close_trade_price(rate=0.00001234) == 0.0011200318470471794
+    assert trade.calc_close_trade_value(rate=0.00001234) == 0.0011200318470471794
 
     # Get the close rate price with a custom close rate and a custom fee rate
-    assert trade.calc_close_trade_price(rate=0.00001234, fee=0.003) == 0.0011194704275749754
+    assert trade.calc_close_trade_value(rate=0.00001234, fee=0.003) == 0.0011194704275749754
 
     # Test when we apply a Sell order, and ask price with a custom fee rate
     trade.update(limit_sell_order)
-    assert trade.calc_close_trade_price(fee=0.005) == 0.0010619972701635854
+    assert trade.calc_close_trade_value(fee=0.005) == 0.0010619972701635854
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -421,9 +456,9 @@ def test_migrate_old(mocker, default_conf, fee):
                                 PRIMARY KEY (id),
                                 CHECK (is_open IN (0, 1))
                                 );"""
-    insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
+    insert_table_old = """INSERT INTO trades (exchange, pair, is_open, open_order_id, fee,
                           open_rate, stake_amount, amount, open_date)
-                          VALUES ('BITTREX', 'BTC_ETC', 1, {fee},
+                          VALUES ('BITTREX', 'BTC_ETC', 1, '123123', {fee},
                           0.00258580, {stake}, {amount},
                           '2017-11-28 12:44:24.000000')
                           """.format(fee=fee.return_value,
@@ -440,14 +475,14 @@ def test_migrate_old(mocker, default_conf, fee):
                                      amount=amount
                                      )
     engine = create_engine('sqlite://')
-    mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine)
+    mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
 
     # Create table using the old format
     engine.execute(create_table_old)
     engine.execute(insert_table_old)
     engine.execute(insert_table_old2)
     # Run init to test migration
-    init(default_conf['db_url'], default_conf['dry_run'])
+    init_db(default_conf['db_url'], default_conf['dry_run'])
 
     assert len(Trade.query.filter(Trade.id == 1).all()) == 1
     trade = Trade.query.filter(Trade.id == 1).first()
@@ -457,18 +492,20 @@ def test_migrate_old(mocker, default_conf, fee):
     assert trade.close_rate_requested is None
     assert trade.is_open == 1
     assert trade.amount == amount
+    assert trade.amount_requested == amount
     assert trade.stake_amount == default_conf.get("stake_amount")
     assert trade.pair == "ETC/BTC"
     assert trade.exchange == "bittrex"
     assert trade.max_rate == 0.0
     assert trade.stop_loss == 0.0
     assert trade.initial_stop_loss == 0.0
-    assert trade.open_trade_price == trade._calc_open_trade_price()
+    assert trade.open_trade_value == trade._calc_open_trade_value()
     assert trade.close_profit_abs is None
     assert trade.fee_open_cost is None
     assert trade.fee_open_currency is None
     assert trade.fee_close_cost is None
     assert trade.fee_close_currency is None
+    assert trade.timeframe is None
 
     trade = Trade.query.filter(Trade.id == 2).first()
     assert trade.close_rate is not None
@@ -479,6 +516,12 @@ def test_migrate_old(mocker, default_conf, fee):
     assert pytest.approx(trade.close_profit_abs) == trade.calc_profit()
     assert trade.sell_order_status is None
 
+    # Should've created one order
+    assert len(Order.query.all()) == 1
+    order = Order.query.first()
+    assert order.order_id == '123123'
+    assert order.ft_order_side == 'buy'
+
 
 def test_migrate_new(mocker, default_conf, fee, caplog):
     """
@@ -507,22 +550,25 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
                                 sell_reason VARCHAR,
                                 strategy VARCHAR,
                                 ticker_interval INTEGER,
+                                stoploss_order_id VARCHAR,
                                 PRIMARY KEY (id),
                                 CHECK (is_open IN (0, 1))
                                 );"""
     insert_table_old = """INSERT INTO trades (exchange, pair, is_open, fee,
                           open_rate, stake_amount, amount, open_date,
-                          stop_loss, initial_stop_loss, max_rate)
+                          stop_loss, initial_stop_loss, max_rate, ticker_interval,
+                          open_order_id, stoploss_order_id)
                           VALUES ('binance', 'ETC/BTC', 1, {fee},
                           0.00258580, {stake}, {amount},
                           '2019-11-28 12:44:24.000000',
-                          0.0, 0.0, 0.0)
+                          0.0, 0.0, 0.0, '5m',
+                          'buy_order', 'stop_order_id222')
                           """.format(fee=fee.return_value,
                                      stake=default_conf.get("stake_amount"),
                                      amount=amount
                                      )
     engine = create_engine('sqlite://')
-    mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine)
+    mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
 
     # Create table using the old format
     engine.execute(create_table_old)
@@ -535,7 +581,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
 
     engine.execute("create table trades_bak1 as select * from trades")
     # Run init to test migration
-    init(default_conf['db_url'], default_conf['dry_run'])
+    init_db(default_conf['db_url'], default_conf['dry_run'])
 
     assert len(Trade.query.filter(Trade.id == 1).all()) == 1
     trade = Trade.query.filter(Trade.id == 1).first()
@@ -545,6 +591,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert trade.close_rate_requested is None
     assert trade.is_open == 1
     assert trade.amount == amount
+    assert trade.amount_requested == amount
     assert trade.stake_amount == default_conf.get("stake_amount")
     assert trade.pair == "ETC/BTC"
     assert trade.exchange == "binance"
@@ -554,15 +601,24 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert trade.initial_stop_loss == 0.0
     assert trade.sell_reason is None
     assert trade.strategy is None
-    assert trade.ticker_interval is None
-    assert trade.stoploss_order_id is None
+    assert trade.timeframe == '5m'
+    assert trade.stoploss_order_id == 'stop_order_id222'
     assert trade.stoploss_last_update is None
     assert log_has("trying trades_bak1", caplog)
     assert log_has("trying trades_bak2", caplog)
-    assert log_has("Running database migration - backup available as trades_bak2", caplog)
-    assert trade.open_trade_price == trade._calc_open_trade_price()
+    assert log_has("Running database migration for trades - backup: trades_bak2", caplog)
+    assert trade.open_trade_value == trade._calc_open_trade_value()
     assert trade.close_profit_abs is None
 
+    assert log_has("Moving open orders to Orders table.", caplog)
+    orders = Order.query.all()
+    assert len(orders) == 2
+    assert orders[0].order_id == 'buy_order'
+    assert orders[0].ft_order_side == 'buy'
+
+    assert orders[1].order_id == 'stop_order_id222'
+    assert orders[1].ft_order_side == 'stoploss'
+
 
 def test_migrate_mid_state(mocker, default_conf, fee, caplog):
     """
@@ -598,14 +654,14 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
                                      amount=amount
                                      )
     engine = create_engine('sqlite://')
-    mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine)
+    mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine)
 
     # Create table using the old format
     engine.execute(create_table_old)
     engine.execute(insert_table_old)
 
     # Run init to test migration
-    init(default_conf['db_url'], default_conf['dry_run'])
+    init_db(default_conf['db_url'], default_conf['dry_run'])
 
     assert len(Trade.query.filter(Trade.id == 1).all()) == 1
     trade = Trade.query.filter(Trade.id == 1).first()
@@ -621,9 +677,9 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
     assert trade.max_rate == 0.0
     assert trade.stop_loss == 0.0
     assert trade.initial_stop_loss == 0.0
-    assert trade.open_trade_price == trade._calc_open_trade_price()
+    assert trade.open_trade_value == trade._calc_open_trade_value()
     assert log_has("trying trades_bak0", caplog)
-    assert log_has("Running database migration - backup available as trades_bak0", caplog)
+    assert log_has("Running database migration for trades - backup: trades_bak0", caplog)
 
 
 def test_adjust_stop_loss(fee):
@@ -710,10 +766,10 @@ def test_adjust_min_max_rates(fee):
 
 
 @pytest.mark.usefixtures("init_persistence")
-def test_get_open(default_conf, fee):
+def test_get_open(fee):
 
     create_mock_trades(fee)
-    assert len(Trade.get_open_trades()) == 2
+    assert len(Trade.get_open_trades()) == 4
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -724,6 +780,7 @@ def test_to_json(default_conf, fee):
         pair='ETH/BTC',
         stake_amount=0.001,
         amount=123.0,
+        amount_requested=123.0,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_date=arrow.utcnow().shift(hours=-2).datetime,
@@ -746,7 +803,7 @@ def test_to_json(default_conf, fee):
                       'close_timestamp': None,
                       'open_rate': 0.123,
                       'open_rate_requested': None,
-                      'open_trade_price': 15.1668225,
+                      'open_trade_value': 15.1668225,
                       'fee_close': 0.0025,
                       'fee_close_cost': None,
                       'fee_close_currency': None,
@@ -756,26 +813,29 @@ def test_to_json(default_conf, fee):
                       'close_rate': None,
                       'close_rate_requested': None,
                       'amount': 123.0,
+                      'amount_requested': 123.0,
                       'stake_amount': 0.001,
                       'close_profit': None,
+                      'close_profit_pct': None,
                       'close_profit_abs': None,
+                      'profit_ratio': None,
+                      'profit_pct': None,
+                      'profit_abs': None,
                       'sell_reason': None,
                       'sell_order_status': None,
-                      'stop_loss': None,
                       'stop_loss_abs': None,
                       'stop_loss_ratio': None,
                       'stop_loss_pct': None,
                       'stoploss_order_id': None,
                       'stoploss_last_update': None,
                       'stoploss_last_update_timestamp': None,
-                      'initial_stop_loss': None,
                       'initial_stop_loss_abs': None,
                       'initial_stop_loss_pct': None,
                       'initial_stop_loss_ratio': None,
                       'min_rate': None,
                       'max_rate': None,
                       'strategy': None,
-                      'ticker_interval': None,
+                      'timeframe': None,
                       'exchange': 'bittrex',
                       }
 
@@ -784,6 +844,7 @@ def test_to_json(default_conf, fee):
         pair='XRP/BTC',
         stake_amount=0.001,
         amount=100.0,
+        amount_requested=101.0,
         fee_open=fee.return_value,
         fee_close=fee.return_value,
         open_date=arrow.utcnow().shift(hours=-2).datetime,
@@ -806,20 +867,23 @@ def test_to_json(default_conf, fee):
                       'open_rate': 0.123,
                       'close_rate': 0.125,
                       'amount': 100.0,
+                      'amount_requested': 101.0,
                       'stake_amount': 0.001,
-                      'stop_loss': None,
                       'stop_loss_abs': None,
                       'stop_loss_pct': None,
                       'stop_loss_ratio': None,
                       'stoploss_order_id': None,
                       'stoploss_last_update': None,
                       'stoploss_last_update_timestamp': None,
-                      'initial_stop_loss': None,
                       'initial_stop_loss_abs': None,
                       'initial_stop_loss_pct': None,
                       'initial_stop_loss_ratio': None,
                       'close_profit': None,
+                      'close_profit_pct': None,
                       'close_profit_abs': None,
+                      'profit_ratio': None,
+                      'profit_pct': None,
+                      'profit_abs': None,
                       'close_rate_requested': None,
                       'fee_close': 0.0025,
                       'fee_close_cost': None,
@@ -832,17 +896,17 @@ def test_to_json(default_conf, fee):
                       'min_rate': None,
                       'open_order_id': None,
                       'open_rate_requested': None,
-                      'open_trade_price': 12.33075,
+                      'open_trade_value': 12.33075,
                       'sell_reason': None,
                       'sell_order_status': None,
                       'strategy': None,
-                      'ticker_interval': None,
+                      'timeframe': None,
                       'exchange': 'bittrex',
                       }
 
 
 def test_stoploss_reinitialization(default_conf, fee):
-    init(default_conf['db_url'])
+    init_db(default_conf['db_url'])
     trade = Trade(
         pair='ETH/BTC',
         stake_amount=0.001,
@@ -977,7 +1041,7 @@ def test_total_open_trades_stakes(fee):
     assert res == 0
     create_mock_trades(fee)
     res = Trade.total_open_trades_stakes()
-    assert res == 0.002
+    assert res == 0.004
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -986,7 +1050,7 @@ def test_get_overall_performance(fee):
     create_mock_trades(fee)
     res = Trade.get_overall_performance()
 
-    assert len(res) == 1
+    assert len(res) == 2
     assert 'pair' in res[0]
     assert 'profit' in res[0]
     assert 'count' in res[0]
@@ -1001,5 +1065,98 @@ def test_get_best_pair(fee):
     create_mock_trades(fee)
     res = Trade.get_best_pair()
     assert len(res) == 2
-    assert res[0] == 'ETC/BTC'
-    assert res[1] == 0.005
+    assert res[0] == 'XRP/BTC'
+    assert res[1] == 0.01
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_update_order_from_ccxt():
+    # Most basic order return (only has orderid)
+    o = Order.parse_from_ccxt_object({'id': '1234'}, 'ETH/BTC', 'buy')
+    assert isinstance(o, Order)
+    assert o.ft_pair == 'ETH/BTC'
+    assert o.ft_order_side == 'buy'
+    assert o.order_id == '1234'
+    assert o.ft_is_open
+    ccxt_order = {
+        'id': '1234',
+        'side': 'buy',
+        'symbol': 'ETH/BTC',
+        'type': 'limit',
+        'price': 1234.5,
+        'amount':  20.0,
+        'filled': 9,
+        'remaining': 11,
+        'status': 'open',
+        'timestamp': 1599394315123
+    }
+    o = Order.parse_from_ccxt_object(ccxt_order, 'ETH/BTC', 'buy')
+    assert isinstance(o, Order)
+    assert o.ft_pair == 'ETH/BTC'
+    assert o.ft_order_side == 'buy'
+    assert o.order_id == '1234'
+    assert o.order_type == 'limit'
+    assert o.price == 1234.5
+    assert o.filled == 9
+    assert o.remaining == 11
+    assert o.order_date is not None
+    assert o.ft_is_open
+    assert o.order_filled_date is None
+
+    # Order has been closed
+    ccxt_order.update({'filled': 20.0, 'remaining': 0.0, 'status': 'closed'})
+    o.update_from_ccxt_object(ccxt_order)
+
+    assert o.filled == 20.0
+    assert o.remaining == 0.0
+    assert not o.ft_is_open
+    assert o.order_filled_date is not None
+
+    ccxt_order.update({'id': 'somethingelse'})
+    with pytest.raises(DependencyException, match=r"Order-id's don't match"):
+        o.update_from_ccxt_object(ccxt_order)
+
+
+@pytest.mark.usefixtures("init_persistence")
+def test_select_order(fee):
+    create_mock_trades(fee)
+
+    trades = Trade.get_trades().all()
+
+    # Open buy order, no sell order
+    order = trades[0].select_order('buy', True)
+    assert order is None
+    order = trades[0].select_order('buy', False)
+    assert order is not None
+    order = trades[0].select_order('sell', None)
+    assert order is None
+
+    # closed buy order, and open sell order
+    order = trades[1].select_order('buy', True)
+    assert order is None
+    order = trades[1].select_order('buy', False)
+    assert order is not None
+    order = trades[1].select_order('buy', None)
+    assert order is not None
+    order = trades[1].select_order('sell', True)
+    assert order is None
+    order = trades[1].select_order('sell', False)
+    assert order is not None
+
+    # Has open buy order
+    order = trades[3].select_order('buy', True)
+    assert order is not None
+    order = trades[3].select_order('buy', False)
+    assert order is None
+
+    # Open sell order
+    order = trades[4].select_order('buy', True)
+    assert order is None
+    order = trades[4].select_order('buy', False)
+    assert order is not None
+
+    order = trades[4].select_order('sell', True)
+    assert order is not None
+    assert order.ft_order_side == 'stoploss'
+    order = trades[4].select_order('sell', False)
+    assert order is None
diff --git a/tests/test_plotting.py b/tests/test_plotting.py
index 150329c52..42847ca50 100644
--- a/tests/test_plotting.py
+++ b/tests/test_plotting.py
@@ -13,15 +13,12 @@ from freqtrade.configuration import TimeRange
 from freqtrade.data import history
 from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data
 from freqtrade.exceptions import OperationalException
-from freqtrade.plot.plotting import (add_indicators, add_profit,
-                                     create_plotconfig,
-                                     generate_candlestick_graph,
-                                     generate_plot_filename,
-                                     generate_profit_graph, init_plotscript,
-                                     load_and_plot_trades, plot_profit,
-                                     plot_trades, store_plot_file)
+from freqtrade.plot.plotting import (add_areas, add_indicators, add_profit, create_plotconfig,
+                                     generate_candlestick_graph, generate_plot_filename,
+                                     generate_profit_graph, init_plotscript, load_and_plot_trades,
+                                     plot_profit, plot_trades, store_plot_file)
 from freqtrade.resolvers import StrategyResolver
-from tests.conftest import get_args, log_has, log_has_re
+from tests.conftest import get_args, log_has, log_has_re, patch_exchange
 
 
 def fig_generating_mock(fig, *args, **kwargs):
@@ -47,16 +44,17 @@ def generate_empty_figure():
 def test_init_plotscript(default_conf, mocker, testdatadir):
     default_conf['timerange'] = "20180110-20180112"
     default_conf['trade_source'] = "file"
-    default_conf['ticker_interval'] = "5m"
+    default_conf['timeframe'] = "5m"
     default_conf["datadir"] = testdatadir
     default_conf['exportfilename'] = testdatadir / "backtest-result_test.json"
     ret = init_plotscript(default_conf)
     assert "ohlcv" in ret
     assert "trades" in ret
     assert "pairs" in ret
+    assert 'timerange' in ret
 
     default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"]
-    ret = init_plotscript(default_conf)
+    ret = init_plotscript(default_conf, 20)
     assert "ohlcv" in ret
     assert "TRX/BTC" in ret["ohlcv"]
     assert "ADA/BTC" in ret["ohlcv"]
@@ -98,6 +96,62 @@ def test_add_indicators(default_conf, testdatadir, caplog):
     assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog)
 
 
+def test_add_areas(default_conf, testdatadir, caplog):
+    pair = "UNITTEST/BTC"
+    timerange = TimeRange(None, 'line', 0, -1000)
+
+    data = history.load_pair_history(pair=pair, timeframe='1m',
+                                     datadir=testdatadir, timerange=timerange)
+    indicators = {"macd": {"color": "red",
+                           "fill_color": "black",
+                           "fill_to": "macdhist",
+                           "fill_label": "MACD Fill"}}
+
+    ind_no_label = {"macd": {"fill_color": "red",
+                             "fill_to": "macdhist"}}
+
+    ind_plain = {"macd": {"fill_to": "macdhist"}}
+    default_conf.update({'strategy': 'DefaultStrategy'})
+    strategy = StrategyResolver.load_strategy(default_conf)
+
+    # Generate buy/sell signals and indicators
+    data = strategy.analyze_ticker(data, {'pair': pair})
+    fig = generate_empty_figure()
+
+    # indicator mentioned in fill_to does not exist
+    fig1 = add_areas(fig, 1, data, {'ema10': {'fill_to': 'no_fill_indicator'}})
+    assert fig == fig1
+    assert log_has_re(r'fill_to: "no_fill_indicator" ignored\..*', caplog)
+
+    # indicator does not exist
+    fig2 = add_areas(fig, 1, data, {'no_indicator': {'fill_to': 'ema10'}})
+    assert fig == fig2
+    assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog)
+
+    # everythin given in plot config, row 3
+    fig3 = add_areas(fig, 3, data, indicators)
+    figure = fig3.layout.figure
+    fill_macd = find_trace_in_fig_data(figure.data, "MACD Fill")
+    assert isinstance(fill_macd, go.Scatter)
+    assert fill_macd.yaxis == "y3"
+    assert fill_macd.fillcolor == "black"
+
+    # label missing, row 1
+    fig4 = add_areas(fig, 1, data, ind_no_label)
+    figure = fig4.layout.figure
+    fill_macd = find_trace_in_fig_data(figure.data, "macd<>macdhist")
+    assert isinstance(fill_macd, go.Scatter)
+    assert fill_macd.yaxis == "y"
+    assert fill_macd.fillcolor == "red"
+
+    # fit_to only
+    fig5 = add_areas(fig, 1, data, ind_plain)
+    figure = fig5.layout.figure
+    fill_macd = find_trace_in_fig_data(figure.data, "macd<>macdhist")
+    assert isinstance(fill_macd, go.Scatter)
+    assert fill_macd.yaxis == "y"
+
+
 def test_plot_trades(testdatadir, caplog):
     fig1 = generate_empty_figure()
     # nothing happens when no trades are available
@@ -267,7 +321,7 @@ def test_generate_profit_graph(testdatadir):
     trades = load_backtest_data(filename)
     timerange = TimeRange.parse_timerange("20180110-20180112")
     pairs = ["TRX/BTC", "XLM/BTC"]
-    trades = trades[trades['close_time'] < pd.Timestamp('2018-01-12', tz='UTC')]
+    trades = trades[trades['close_date'] < pd.Timestamp('2018-01-12', tz='UTC')]
 
     data = history.load_data(datadir=testdatadir,
                              pairs=pairs,
@@ -316,6 +370,8 @@ def test_start_plot_dataframe(mocker):
 
 
 def test_load_and_plot_trades(default_conf, mocker, caplog, testdatadir):
+    patch_exchange(mocker)
+
     default_conf['trade_source'] = 'file'
     default_conf["datadir"] = testdatadir
     default_conf['exportfilename'] = testdatadir / "backtest-result_test.json"
@@ -360,22 +416,22 @@ def test_start_plot_profit(mocker):
 def test_start_plot_profit_error(mocker):
 
     args = [
-        "plot-profit",
-        "--pairs", "ETH/BTC"
+        'plot-profit',
+        '--pairs', 'ETH/BTC'
     ]
     argsp = get_args(args)
     # Make sure we use no config. Details: #2241
     # not resetting config causes random failures if config.json exists
-    argsp["config"] = []
+    argsp['config'] = []
     with pytest.raises(OperationalException):
         start_plot_profit(argsp)
 
 
 def test_plot_profit(default_conf, mocker, testdatadir, caplog):
     default_conf['trade_source'] = 'file'
-    default_conf["datadir"] = testdatadir
-    default_conf['exportfilename'] = testdatadir / "backtest-result_test_nofile.json"
-    default_conf['pairs'] = ["ETH/BTC", "LTC/BTC"]
+    default_conf['datadir'] = testdatadir
+    default_conf['exportfilename'] = testdatadir / 'backtest-result_test_nofile.json'
+    default_conf['pairs'] = ['ETH/BTC', 'LTC/BTC']
 
     profit_mock = MagicMock()
     store_mock = MagicMock()
diff --git a/tests/test_talib.py b/tests/test_talib.py
index 2c7f73eb1..f526fdd4d 100644
--- a/tests/test_talib.py
+++ b/tests/test_talib.py
@@ -1,7 +1,5 @@
-
-
-import talib.abstract as ta
 import pandas as pd
+import talib.abstract as ta
 
 
 def test_talib_bollingerbands_near_zero_values():
diff --git a/tests/test_wallets.py b/tests/test_wallets.py
index 884470014..b7aead0c4 100644
--- a/tests/test_wallets.py
+++ b/tests/test_wallets.py
@@ -19,12 +19,17 @@ def test_sync_wallet_at_boot(mocker, default_conf):
                 "used": 0.0,
                 "total": 0.260739
             },
+            "USDT": {
+                "free": 20,
+                "used": 20,
+                "total": 40
+            },
         })
     )
 
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
-    assert len(freqtrade.wallets._wallets) == 2
+    assert len(freqtrade.wallets._wallets) == 3
     assert freqtrade.wallets._wallets['BNT'].free == 1.0
     assert freqtrade.wallets._wallets['BNT'].used == 2.0
     assert freqtrade.wallets._wallets['BNT'].total == 3.0
@@ -32,6 +37,7 @@ def test_sync_wallet_at_boot(mocker, default_conf):
     assert freqtrade.wallets._wallets['GAS'].used == 0.0
     assert freqtrade.wallets._wallets['GAS'].total == 0.260739
     assert freqtrade.wallets.get_free('BNT') == 1.0
+    assert 'USDT' in freqtrade.wallets._wallets
     assert freqtrade.wallets._last_wallet_refresh > 0
     mocker.patch.multiple(
         'freqtrade.exchange.Exchange',
@@ -51,6 +57,7 @@ def test_sync_wallet_at_boot(mocker, default_conf):
 
     freqtrade.wallets.update()
 
+    # USDT is missing from the 2nd result - so should not be in this either.
     assert len(freqtrade.wallets._wallets) == 2
     assert freqtrade.wallets._wallets['BNT'].free == 1.2
     assert freqtrade.wallets._wallets['BNT'].used == 1.9
@@ -67,6 +74,10 @@ def test_sync_wallet_at_boot(mocker, default_conf):
     freqtrade.wallets.update()
     assert update_mock.call_count == 1
 
+    assert freqtrade.wallets.get_free('NOCURRENCY') == 0
+    assert freqtrade.wallets.get_used('NOCURRENCY') == 0
+    assert freqtrade.wallets.get_total('NOCURRENCY') == 0
+
 
 def test_sync_wallet_missing_data(mocker, default_conf):
     default_conf['dry_run'] = False
diff --git a/tests/testdata/.last_result.json b/tests/testdata/.last_result.json
new file mode 100644
index 000000000..98448e10f
--- /dev/null
+++ b/tests/testdata/.last_result.json
@@ -0,0 +1 @@
+{"latest_backtest":"backtest-result_new.json"}
diff --git a/tests/testdata/UNITTEST_BTC-5m.h5 b/tests/testdata/UNITTEST_BTC-5m.h5
new file mode 100644
index 000000000..52232af9e
Binary files /dev/null and b/tests/testdata/UNITTEST_BTC-5m.h5 differ
diff --git a/tests/testdata/XRP_ETH-trades.h5 b/tests/testdata/XRP_ETH-trades.h5
new file mode 100644
index 000000000..c13789e2a
Binary files /dev/null and b/tests/testdata/XRP_ETH-trades.h5 differ
diff --git a/tests/testdata/backtest-result_multistrat.json b/tests/testdata/backtest-result_multistrat.json
new file mode 100644
index 000000000..0e5386ef3
--- /dev/null
+++ b/tests/testdata/backtest-result_multistrat.json
@@ -0,0 +1 @@
+{"strategy": {"DefaultStrategy": {"trades": [{"pair": "TRX/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:15:00+00:00", "close_date": "2018-01-10 07:20:00+00:00", "trade_duration": 5, "open_rate": 9.64e-05, "close_rate": 0.00010074887218045112, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1037.344398340249, "profit_abs": 0.00399999999999999}, {"pair": "ADA/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:15:00+00:00", "close_date": "2018-01-10 07:30:00+00:00", "trade_duration": 15, "open_rate": 4.756e-05, "close_rate": 4.9705563909774425e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2102.6072329688814, "profit_abs": 0.00399999999999999}, {"pair": "XLM/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:25:00+00:00", "close_date": "2018-01-10 07:35:00+00:00", "trade_duration": 10, "open_rate": 3.339e-05, "close_rate": 3.489631578947368e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2994.908655286014, "profit_abs": 0.0040000000000000036}, {"pair": "TRX/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:25:00+00:00", "close_date": "2018-01-10 07:40:00+00:00", "trade_duration": 15, "open_rate": 9.696e-05, "close_rate": 0.00010133413533834584, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1031.3531353135315, "profit_abs": 0.00399999999999999}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 07:35:00+00:00", "close_date": "2018-01-10 08:35:00+00:00", "trade_duration": 60, "open_rate": 0.0943, "close_rate": 0.09477268170426063, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0604453870625663, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-10 07:40:00+00:00", "close_date": "2018-01-10 08:10:00+00:00", "trade_duration": 30, "open_rate": 0.02719607, "close_rate": 0.02760503345864661, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.677001860930642, "profit_abs": 0.0010000000000000009}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-10 08:15:00+00:00", "close_date": "2018-01-10 09:55:00+00:00", "trade_duration": 100, "open_rate": 0.04634952, "close_rate": 0.046581848421052625, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.1575196463739, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 14:45:00+00:00", "close_date": "2018-01-10 15:50:00+00:00", "trade_duration": 65, "open_rate": 3.066e-05, "close_rate": 3.081368421052631e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3261.5786040443577, "profit_abs": -1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-10 16:35:00+00:00", "close_date": "2018-01-10 17:15:00+00:00", "trade_duration": 40, "open_rate": 0.0168999, "close_rate": 0.016984611278195488, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 5.917194776300452, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 16:40:00+00:00", "close_date": "2018-01-10 17:20:00+00:00", "trade_duration": 40, "open_rate": 0.09132568, "close_rate": 0.0917834528320802, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0949822656672252, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 18:50:00+00:00", "close_date": "2018-01-10 19:45:00+00:00", "trade_duration": 55, "open_rate": 0.08898003, "close_rate": 0.08942604518796991, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1238476768326557, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-10 22:15:00+00:00", "close_date": "2018-01-10 23:00:00+00:00", "trade_duration": 45, "open_rate": 0.08560008, "close_rate": 0.08602915308270676, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1682232072680307, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-10 22:50:00+00:00", "close_date": "2018-01-10 23:20:00+00:00", "trade_duration": 30, "open_rate": 0.00249083, "close_rate": 0.0025282860902255634, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 40.147260150231055, "profit_abs": 0.000999999999999987}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 23:15:00+00:00", "close_date": "2018-01-11 00:15:00+00:00", "trade_duration": 60, "open_rate": 3.022e-05, "close_rate": 3.037147869674185e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3309.0668431502318, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-10 23:40:00+00:00", "close_date": "2018-01-11 00:05:00+00:00", "trade_duration": 25, "open_rate": 0.002437, "close_rate": 0.0024980776942355883, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 41.03405826836274, "profit_abs": 0.001999999999999974}, {"pair": "ZEC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 00:00:00+00:00", "close_date": "2018-01-11 00:35:00+00:00", "trade_duration": 35, "open_rate": 0.04771803, "close_rate": 0.04843559436090225, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.0956439316543456, "profit_abs": 0.0010000000000000009}, {"pair": "XLM/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-11 03:40:00+00:00", "close_date": "2018-01-11 04:25:00+00:00", "trade_duration": 45, "open_rate": 3.651e-05, "close_rate": 3.2859000000000005e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2738.9756231169545, "profit_abs": -0.01047499999999997}, {"pair": "ETH/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 03:55:00+00:00", "close_date": "2018-01-11 04:25:00+00:00", "trade_duration": 30, "open_rate": 0.08824105, "close_rate": 0.08956798308270676, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1332594070446804, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 04:00:00+00:00", "close_date": "2018-01-11 04:50:00+00:00", "trade_duration": 50, "open_rate": 0.00243, "close_rate": 0.002442180451127819, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 41.1522633744856, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 04:30:00+00:00", "close_date": "2018-01-11 04:55:00+00:00", "trade_duration": 25, "open_rate": 0.04545064, "close_rate": 0.046589753784461146, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.200189040242338, "profit_abs": 0.001999999999999988}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 04:30:00+00:00", "close_date": "2018-01-11 04:50:00+00:00", "trade_duration": 20, "open_rate": 3.372e-05, "close_rate": 3.456511278195488e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2965.599051008304, "profit_abs": 0.001999999999999988}, {"pair": "XMR/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 04:55:00+00:00", "close_date": "2018-01-11 05:15:00+00:00", "trade_duration": 20, "open_rate": 0.02644, "close_rate": 0.02710265664160401, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.7821482602118004, "profit_abs": 0.001999999999999988}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 11:20:00+00:00", "close_date": "2018-01-11 12:00:00+00:00", "trade_duration": 40, "open_rate": 0.08812, "close_rate": 0.08856170426065162, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1348161597821154, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 11:35:00+00:00", "close_date": "2018-01-11 12:15:00+00:00", "trade_duration": 40, "open_rate": 0.02683577, "close_rate": 0.026970285137844607, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.7263696923919087, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 14:00:00+00:00", "close_date": "2018-01-11 14:25:00+00:00", "trade_duration": 25, "open_rate": 4.919e-05, "close_rate": 5.04228320802005e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2032.9335230737956, "profit_abs": 0.0020000000000000018}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 19:25:00+00:00", "close_date": "2018-01-11 20:35:00+00:00", "trade_duration": 70, "open_rate": 0.08784896, "close_rate": 0.08828930566416039, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1383174029607181, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 22:35:00+00:00", "close_date": "2018-01-11 23:30:00+00:00", "trade_duration": 55, "open_rate": 5.105e-05, "close_rate": 5.130588972431077e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1958.8638589618022, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 22:55:00+00:00", "close_date": "2018-01-11 23:25:00+00:00", "trade_duration": 30, "open_rate": 3.96e-05, "close_rate": 4.019548872180451e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2525.252525252525, "profit_abs": 0.0010000000000000148}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 22:55:00+00:00", "close_date": "2018-01-11 23:35:00+00:00", "trade_duration": 40, "open_rate": 2.885e-05, "close_rate": 2.899461152882205e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3466.204506065858, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 23:30:00+00:00", "close_date": "2018-01-12 00:05:00+00:00", "trade_duration": 35, "open_rate": 0.02645, "close_rate": 0.026847744360902256, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.780718336483932, "profit_abs": 0.0010000000000000148}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 23:55:00+00:00", "close_date": "2018-01-12 01:15:00+00:00", "trade_duration": 80, "open_rate": 0.048, "close_rate": 0.04824060150375939, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.0833333333333335, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-12 21:15:00+00:00", "close_date": "2018-01-12 21:40:00+00:00", "trade_duration": 25, "open_rate": 4.692e-05, "close_rate": 4.809593984962405e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2131.287297527707, "profit_abs": 0.001999999999999974}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 00:55:00+00:00", "close_date": "2018-01-13 06:20:00+00:00", "trade_duration": 325, "open_rate": 0.00256966, "close_rate": 0.0025825405012531327, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.91565421106294, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.0, "open_date": "2018-01-13 10:55:00+00:00", "close_date": "2018-01-13 11:35:00+00:00", "trade_duration": 40, "open_rate": 6.262e-05, "close_rate": 6.293388471177944e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1596.933886937081, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-13 13:05:00+00:00", "close_date": "2018-01-15 14:10:00+00:00", "trade_duration": 2945, "open_rate": 4.73e-05, "close_rate": 4.753709273182957e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2114.1649048625795, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 13:30:00+00:00", "close_date": "2018-01-13 14:45:00+00:00", "trade_duration": 75, "open_rate": 6.063e-05, "close_rate": 6.0933909774436085e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1649.348507339601, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 13:40:00+00:00", "close_date": "2018-01-13 23:30:00+00:00", "trade_duration": 590, "open_rate": 0.00011082, "close_rate": 0.00011137548872180448, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 902.3641941887746, "profit_abs": -2.7755575615628914e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 15:15:00+00:00", "close_date": "2018-01-13 15:55:00+00:00", "trade_duration": 40, "open_rate": 5.93e-05, "close_rate": 5.9597243107769415e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1686.3406408094436, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 16:30:00+00:00", "close_date": "2018-01-13 17:10:00+00:00", "trade_duration": 40, "open_rate": 0.04850003, "close_rate": 0.04874313791979949, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.0618543947292407, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 22:05:00+00:00", "close_date": "2018-01-14 06:25:00+00:00", "trade_duration": 500, "open_rate": 0.09825019, "close_rate": 0.09874267215538848, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0178097365511456, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": 0.0, "open_date": "2018-01-14 00:20:00+00:00", "close_date": "2018-01-14 22:55:00+00:00", "trade_duration": 1355, "open_rate": 6.018e-05, "close_rate": 6.048165413533834e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1661.681621801263, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-14 12:45:00+00:00", "close_date": "2018-01-14 13:25:00+00:00", "trade_duration": 40, "open_rate": 0.09758999, "close_rate": 0.0980791628822055, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.024695258191952, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-14 15:30:00+00:00", "close_date": "2018-01-14 16:00:00+00:00", "trade_duration": 30, "open_rate": 0.00311, "close_rate": 0.0031567669172932328, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 32.154340836012864, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-14 20:45:00+00:00", "close_date": "2018-01-14 22:15:00+00:00", "trade_duration": 90, "open_rate": 0.00312401, "close_rate": 0.003139669197994987, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 32.010140812609436, "profit_abs": -1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-14 23:35:00+00:00", "close_date": "2018-01-15 00:30:00+00:00", "trade_duration": 55, "open_rate": 0.0174679, "close_rate": 0.017555458395989976, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 5.724786608579165, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-14 23:45:00+00:00", "close_date": "2018-01-15 00:25:00+00:00", "trade_duration": 40, "open_rate": 0.07346846, "close_rate": 0.07383672295739348, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.3611282991367997, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 02:25:00+00:00", "close_date": "2018-01-15 03:05:00+00:00", "trade_duration": 40, "open_rate": 0.097994, "close_rate": 0.09848519799498744, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.020470641059657, "profit_abs": -2.7755575615628914e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 07:20:00+00:00", "close_date": "2018-01-15 08:00:00+00:00", "trade_duration": 40, "open_rate": 0.09659, "close_rate": 0.09707416040100247, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0353038616834043, "profit_abs": -2.7755575615628914e-17}, {"pair": "TRX/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-15 08:20:00+00:00", "close_date": "2018-01-15 08:55:00+00:00", "trade_duration": 35, "open_rate": 9.987e-05, "close_rate": 0.00010137180451127818, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1001.3016921998599, "profit_abs": 0.0010000000000000009}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-15 12:10:00+00:00", "close_date": "2018-01-16 02:50:00+00:00", "trade_duration": 880, "open_rate": 0.0948969, "close_rate": 0.09537257368421052, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0537752023511833, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 14:10:00+00:00", "close_date": "2018-01-15 17:40:00+00:00", "trade_duration": 210, "open_rate": 0.071, "close_rate": 0.07135588972431077, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4084507042253522, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 14:30:00+00:00", "close_date": "2018-01-15 15:10:00+00:00", "trade_duration": 40, "open_rate": 0.04600501, "close_rate": 0.046235611553884705, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.173676301776698, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 18:10:00+00:00", "close_date": "2018-01-15 19:25:00+00:00", "trade_duration": 75, "open_rate": 9.438e-05, "close_rate": 9.485308270676693e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1059.5465140919687, "profit_abs": 1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 18:35:00+00:00", "close_date": "2018-01-15 19:15:00+00:00", "trade_duration": 40, "open_rate": 0.03040001, "close_rate": 0.030552391002506264, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.2894726021471703, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-15 20:25:00+00:00", "close_date": "2018-01-16 08:25:00+00:00", "trade_duration": 720, "open_rate": 5.837e-05, "close_rate": 5.2533e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1713.2088401576154, "profit_abs": -0.010474999999999984}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 20:40:00+00:00", "close_date": "2018-01-15 22:00:00+00:00", "trade_duration": 80, "open_rate": 0.046036, "close_rate": 0.04626675689223057, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.1722130506560084, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-16 00:30:00+00:00", "close_date": "2018-01-16 01:10:00+00:00", "trade_duration": 40, "open_rate": 0.0028685, "close_rate": 0.0028828784461152877, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 34.86142583231654, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": 0.0, "open_date": "2018-01-16 01:15:00+00:00", "close_date": "2018-01-16 02:35:00+00:00", "trade_duration": 80, "open_rate": 0.06731755, "close_rate": 0.0676549813283208, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4854967241083492, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-16 07:45:00+00:00", "close_date": "2018-01-16 08:40:00+00:00", "trade_duration": 55, "open_rate": 0.09217614, "close_rate": 0.09263817578947368, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0848794492804754, "profit_abs": 0.0}, {"pair": "LTC/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 08:35:00+00:00", "close_date": "2018-01-16 08:55:00+00:00", "trade_duration": 20, "open_rate": 0.0165, "close_rate": 0.016913533834586467, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.0606060606060606, "profit_abs": 0.0020000000000000018}, {"pair": "TRX/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 08:35:00+00:00", "close_date": "2018-01-16 08:40:00+00:00", "trade_duration": 5, "open_rate": 7.953e-05, "close_rate": 8.311781954887218e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1257.387149503332, "profit_abs": 0.00399999999999999}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-16 08:45:00+00:00", "close_date": "2018-01-16 09:50:00+00:00", "trade_duration": 65, "open_rate": 0.045202, "close_rate": 0.04542857644110275, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.2122914915269236, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 09:15:00+00:00", "close_date": "2018-01-16 09:45:00+00:00", "trade_duration": 30, "open_rate": 5.248e-05, "close_rate": 5.326917293233082e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1905.487804878049, "profit_abs": 0.0010000000000000009}, {"pair": "XMR/BTC", "profit_percent": 0.0, "open_date": "2018-01-16 09:15:00+00:00", "close_date": "2018-01-16 09:55:00+00:00", "trade_duration": 40, "open_rate": 0.02892318, "close_rate": 0.02906815834586466, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.457434486802627, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 09:50:00+00:00", "close_date": "2018-01-16 10:10:00+00:00", "trade_duration": 20, "open_rate": 5.158e-05, "close_rate": 5.287273182957392e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1938.735944164405, "profit_abs": 0.001999999999999988}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 10:05:00+00:00", "close_date": "2018-01-16 10:35:00+00:00", "trade_duration": 30, "open_rate": 0.02828232, "close_rate": 0.02870761804511278, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.5357778286929786, "profit_abs": 0.0010000000000000009}, {"pair": "ZEC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 10:05:00+00:00", "close_date": "2018-01-16 10:40:00+00:00", "trade_duration": 35, "open_rate": 0.04357584, "close_rate": 0.044231115789473675, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.294849623093898, "profit_abs": 0.0010000000000000009}, {"pair": "ADA/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 13:45:00+00:00", "close_date": "2018-01-16 14:20:00+00:00", "trade_duration": 35, "open_rate": 5.362e-05, "close_rate": 5.442631578947368e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1864.975755315181, "profit_abs": 0.0010000000000000148}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-16 17:30:00+00:00", "close_date": "2018-01-16 18:25:00+00:00", "trade_duration": 55, "open_rate": 5.302e-05, "close_rate": 5.328576441102756e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1886.0807242549984, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 18:15:00+00:00", "close_date": "2018-01-16 18:45:00+00:00", "trade_duration": 30, "open_rate": 0.09129999, "close_rate": 0.09267292218045112, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0952903718828448, "profit_abs": 0.0010000000000000148}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 18:15:00+00:00", "close_date": "2018-01-16 18:35:00+00:00", "trade_duration": 20, "open_rate": 3.808e-05, "close_rate": 3.903438596491228e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2626.0504201680674, "profit_abs": 0.0020000000000000018}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 19:00:00+00:00", "close_date": "2018-01-16 19:30:00+00:00", "trade_duration": 30, "open_rate": 0.02811012, "close_rate": 0.028532828571428567, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.557437677249333, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-16 21:25:00+00:00", "close_date": "2018-01-16 22:25:00+00:00", "trade_duration": 60, "open_rate": 0.00258379, "close_rate": 0.002325411, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.702835756775904, "profit_abs": -0.010474999999999984}, {"pair": "NXT/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-16 21:25:00+00:00", "close_date": "2018-01-16 22:45:00+00:00", "trade_duration": 80, "open_rate": 2.559e-05, "close_rate": 2.3031e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3907.7764751856193, "profit_abs": -0.010474999999999998}, {"pair": "TRX/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-16 21:35:00+00:00", "close_date": "2018-01-16 22:25:00+00:00", "trade_duration": 50, "open_rate": 7.62e-05, "close_rate": 6.858e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1312.3359580052495, "profit_abs": -0.010474999999999984}, {"pair": "ETC/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:30:00+00:00", "close_date": "2018-01-16 22:35:00+00:00", "trade_duration": 5, "open_rate": 0.00229844, "close_rate": 0.002402129022556391, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 43.507770487809125, "profit_abs": 0.004000000000000017}, {"pair": "LTC/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:30:00+00:00", "close_date": "2018-01-16 22:40:00+00:00", "trade_duration": 10, "open_rate": 0.0151, "close_rate": 0.015781203007518795, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.622516556291391, "profit_abs": 0.00399999999999999}, {"pair": "ETC/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:40:00+00:00", "close_date": "2018-01-16 22:45:00+00:00", "trade_duration": 5, "open_rate": 0.00235676, "close_rate": 0.00246308, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 42.431134269081284, "profit_abs": 0.0040000000000000036}, {"pair": "DASH/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 22:45:00+00:00", "close_date": "2018-01-16 23:05:00+00:00", "trade_duration": 20, "open_rate": 0.0630692, "close_rate": 0.06464988170426066, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.585559988076589, "profit_abs": 0.0020000000000000018}, {"pair": "NXT/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:50:00+00:00", "close_date": "2018-01-16 22:55:00+00:00", "trade_duration": 5, "open_rate": 2.2e-05, "close_rate": 2.299248120300751e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 4545.454545454546, "profit_abs": 0.003999999999999976}, {"pair": "ADA/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-17 03:30:00+00:00", "close_date": "2018-01-17 04:00:00+00:00", "trade_duration": 30, "open_rate": 4.974e-05, "close_rate": 5.048796992481203e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2010.454362685967, "profit_abs": 0.0010000000000000009}, {"pair": "TRX/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-17 03:55:00+00:00", "close_date": "2018-01-17 04:15:00+00:00", "trade_duration": 20, "open_rate": 7.108e-05, "close_rate": 7.28614536340852e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1406.8655036578502, "profit_abs": 0.001999999999999974}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 09:35:00+00:00", "close_date": "2018-01-17 10:15:00+00:00", "trade_duration": 40, "open_rate": 0.04327, "close_rate": 0.04348689223057644, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.3110700254217704, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 10:20:00+00:00", "close_date": "2018-01-17 17:00:00+00:00", "trade_duration": 400, "open_rate": 4.997e-05, "close_rate": 5.022047619047618e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2001.2007204322595, "profit_abs": -1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 10:30:00+00:00", "close_date": "2018-01-17 11:25:00+00:00", "trade_duration": 55, "open_rate": 0.06836818, "close_rate": 0.06871087764411027, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4626687444363737, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 10:30:00+00:00", "close_date": "2018-01-17 11:10:00+00:00", "trade_duration": 40, "open_rate": 3.63e-05, "close_rate": 3.648195488721804e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2754.8209366391184, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 12:30:00+00:00", "close_date": "2018-01-17 22:05:00+00:00", "trade_duration": 575, "open_rate": 0.0281, "close_rate": 0.02824085213032581, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.5587188612099645, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 12:35:00+00:00", "close_date": "2018-01-17 16:55:00+00:00", "trade_duration": 260, "open_rate": 0.08651001, "close_rate": 0.08694364413533832, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1559355963546878, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 05:00:00+00:00", "close_date": "2018-01-18 05:55:00+00:00", "trade_duration": 55, "open_rate": 5.633e-05, "close_rate": 5.6612355889724306e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1775.2529735487308, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-18 05:20:00+00:00", "close_date": "2018-01-18 05:55:00+00:00", "trade_duration": 35, "open_rate": 0.06988494, "close_rate": 0.07093584135338346, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.430923457900944, "profit_abs": 0.0010000000000000009}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 07:35:00+00:00", "close_date": "2018-01-18 08:15:00+00:00", "trade_duration": 40, "open_rate": 5.545e-05, "close_rate": 5.572794486215538e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1803.4265103697026, "profit_abs": -1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 09:00:00+00:00", "close_date": "2018-01-18 09:40:00+00:00", "trade_duration": 40, "open_rate": 0.01633527, "close_rate": 0.016417151052631574, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.121723118136401, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 16:40:00+00:00", "close_date": "2018-01-18 17:20:00+00:00", "trade_duration": 40, "open_rate": 0.00269734, "close_rate": 0.002710860501253133, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 37.073561360451414, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-18 18:05:00+00:00", "close_date": "2018-01-18 18:30:00+00:00", "trade_duration": 25, "open_rate": 4.475e-05, "close_rate": 4.587155388471177e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2234.63687150838, "profit_abs": 0.0020000000000000018}, {"pair": "NXT/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-18 18:25:00+00:00", "close_date": "2018-01-18 18:55:00+00:00", "trade_duration": 30, "open_rate": 2.79e-05, "close_rate": 2.8319548872180444e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3584.2293906810037, "profit_abs": 0.000999999999999987}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 20:10:00+00:00", "close_date": "2018-01-18 20:50:00+00:00", "trade_duration": 40, "open_rate": 0.04439326, "close_rate": 0.04461578260651629, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.2525942001105577, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 21:30:00+00:00", "close_date": "2018-01-19 00:35:00+00:00", "trade_duration": 185, "open_rate": 4.49e-05, "close_rate": 4.51250626566416e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2227.1714922049, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 21:55:00+00:00", "close_date": "2018-01-19 05:05:00+00:00", "trade_duration": 430, "open_rate": 0.02855, "close_rate": 0.028693107769423555, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.502626970227671, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 22:10:00+00:00", "close_date": "2018-01-18 22:50:00+00:00", "trade_duration": 40, "open_rate": 5.796e-05, "close_rate": 5.8250526315789473e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1725.3278122843342, "profit_abs": 1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 23:50:00+00:00", "close_date": "2018-01-19 00:30:00+00:00", "trade_duration": 40, "open_rate": 0.04340323, "close_rate": 0.04362079005012531, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.303975994413319, "profit_abs": 1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-19 16:45:00+00:00", "close_date": "2018-01-19 17:35:00+00:00", "trade_duration": 50, "open_rate": 0.04454455, "close_rate": 0.04476783095238095, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.244943545282195, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-19 17:15:00+00:00", "close_date": "2018-01-19 19:55:00+00:00", "trade_duration": 160, "open_rate": 5.62e-05, "close_rate": 5.648170426065162e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1779.3594306049824, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-19 17:20:00+00:00", "close_date": "2018-01-19 20:15:00+00:00", "trade_duration": 175, "open_rate": 4.339e-05, "close_rate": 4.360749373433584e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2304.6784973496196, "profit_abs": -1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": 0.0, "open_date": "2018-01-20 04:45:00+00:00", "close_date": "2018-01-20 17:35:00+00:00", "trade_duration": 770, "open_rate": 0.0001009, "close_rate": 0.00010140576441102755, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 991.0802775024778, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 04:50:00+00:00", "close_date": "2018-01-20 15:15:00+00:00", "trade_duration": 625, "open_rate": 0.00270505, "close_rate": 0.002718609147869674, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 36.96789338459548, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 04:50:00+00:00", "close_date": "2018-01-20 07:00:00+00:00", "trade_duration": 130, "open_rate": 0.03000002, "close_rate": 0.030150396040100245, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.3333311111125927, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 09:00:00+00:00", "close_date": "2018-01-20 09:40:00+00:00", "trade_duration": 40, "open_rate": 5.46e-05, "close_rate": 5.4873684210526304e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1831.5018315018317, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-20 18:25:00+00:00", "close_date": "2018-01-25 03:50:00+00:00", "trade_duration": 6325, "open_rate": 0.03082222, "close_rate": 0.027739998, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.244412634781012, "profit_abs": -0.010474999999999998}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 22:25:00+00:00", "close_date": "2018-01-20 23:15:00+00:00", "trade_duration": 50, "open_rate": 0.08969999, "close_rate": 0.09014961401002504, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1148273260677064, "profit_abs": 0.0}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-21 02:50:00+00:00", "close_date": "2018-01-21 14:30:00+00:00", "trade_duration": 700, "open_rate": 0.01632501, "close_rate": 0.01640683962406015, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.125570520324337, "profit_abs": 1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-21 10:20:00+00:00", "close_date": "2018-01-21 11:00:00+00:00", "trade_duration": 40, "open_rate": 0.070538, "close_rate": 0.07089157393483708, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.417675579120474, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-21 15:50:00+00:00", "close_date": "2018-01-21 18:45:00+00:00", "trade_duration": 175, "open_rate": 5.301e-05, "close_rate": 5.327571428571427e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1886.4365214110546, "profit_abs": -2.7755575615628914e-17}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-21 16:20:00+00:00", "close_date": "2018-01-21 17:00:00+00:00", "trade_duration": 40, "open_rate": 3.955e-05, "close_rate": 3.9748245614035085e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2528.4450063211125, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-21 21:15:00+00:00", "close_date": "2018-01-21 21:45:00+00:00", "trade_duration": 30, "open_rate": 0.00258505, "close_rate": 0.002623922932330827, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.6839712964933, "profit_abs": 0.0010000000000000009}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-21 21:15:00+00:00", "close_date": "2018-01-21 21:55:00+00:00", "trade_duration": 40, "open_rate": 3.903e-05, "close_rate": 3.922563909774435e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2562.1316935690497, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 00:35:00+00:00", "close_date": "2018-01-22 10:35:00+00:00", "trade_duration": 600, "open_rate": 5.236e-05, "close_rate": 5.262245614035087e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1909.8548510313217, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": 0.0, "open_date": "2018-01-22 01:30:00+00:00", "close_date": "2018-01-22 02:10:00+00:00", "trade_duration": 40, "open_rate": 9.028e-05, "close_rate": 9.07325313283208e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1107.6650420912717, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 12:25:00+00:00", "close_date": "2018-01-22 14:35:00+00:00", "trade_duration": 130, "open_rate": 0.002687, "close_rate": 0.002700468671679198, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 37.21622627465575, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 13:15:00+00:00", "close_date": "2018-01-22 13:55:00+00:00", "trade_duration": 40, "open_rate": 4.168e-05, "close_rate": 4.188892230576441e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2399.232245681382, "profit_abs": 1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-22 14:00:00+00:00", "close_date": "2018-01-22 14:30:00+00:00", "trade_duration": 30, "open_rate": 8.821e-05, "close_rate": 8.953646616541353e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1133.6583153837435, "profit_abs": 0.0010000000000000148}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 15:55:00+00:00", "close_date": "2018-01-22 16:40:00+00:00", "trade_duration": 45, "open_rate": 5.172e-05, "close_rate": 5.1979248120300745e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1933.4880123743235, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-22 16:05:00+00:00", "close_date": "2018-01-22 16:25:00+00:00", "trade_duration": 20, "open_rate": 3.026e-05, "close_rate": 3.101839598997494e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3304.692663582287, "profit_abs": 0.0020000000000000157}, {"pair": "DASH/BTC", "profit_percent": 0.0, "open_date": "2018-01-22 19:50:00+00:00", "close_date": "2018-01-23 00:10:00+00:00", "trade_duration": 260, "open_rate": 0.07064, "close_rate": 0.07099408521303258, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.415628539071348, "profit_abs": 1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-22 21:25:00+00:00", "close_date": "2018-01-22 22:05:00+00:00", "trade_duration": 40, "open_rate": 0.01644483, "close_rate": 0.01652726022556391, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.080938507725528, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-23 00:05:00+00:00", "close_date": "2018-01-23 00:35:00+00:00", "trade_duration": 30, "open_rate": 4.331e-05, "close_rate": 4.3961278195488714e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2308.935580697299, "profit_abs": 0.0010000000000000148}, {"pair": "NXT/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-23 01:50:00+00:00", "close_date": "2018-01-23 02:15:00+00:00", "trade_duration": 25, "open_rate": 3.2e-05, "close_rate": 3.2802005012531326e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3125.0000000000005, "profit_abs": 0.0020000000000000018}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 04:25:00+00:00", "close_date": "2018-01-23 05:15:00+00:00", "trade_duration": 50, "open_rate": 0.09167706, "close_rate": 0.09213659413533835, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0907854156754153, "profit_abs": 1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 07:35:00+00:00", "close_date": "2018-01-23 09:00:00+00:00", "trade_duration": 85, "open_rate": 0.0692498, "close_rate": 0.06959691679197995, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4440474918339115, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 10:50:00+00:00", "close_date": "2018-01-23 13:05:00+00:00", "trade_duration": 135, "open_rate": 3.182e-05, "close_rate": 3.197949874686716e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3142.677561282213, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 11:05:00+00:00", "close_date": "2018-01-23 16:05:00+00:00", "trade_duration": 300, "open_rate": 0.04088, "close_rate": 0.04108491228070175, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4461839530332683, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 14:55:00+00:00", "close_date": "2018-01-23 15:35:00+00:00", "trade_duration": 40, "open_rate": 5.15e-05, "close_rate": 5.175814536340851e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1941.747572815534, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 16:35:00+00:00", "close_date": "2018-01-24 00:05:00+00:00", "trade_duration": 450, "open_rate": 0.09071698, "close_rate": 0.09117170170426064, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1023294646713329, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 17:25:00+00:00", "close_date": "2018-01-23 18:45:00+00:00", "trade_duration": 80, "open_rate": 3.128e-05, "close_rate": 3.1436791979949865e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3196.9309462915603, "profit_abs": -2.7755575615628914e-17}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 20:15:00+00:00", "close_date": "2018-01-23 22:00:00+00:00", "trade_duration": 105, "open_rate": 9.555e-05, "close_rate": 9.602894736842104e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1046.5724751439038, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 22:30:00+00:00", "close_date": "2018-01-23 23:10:00+00:00", "trade_duration": 40, "open_rate": 0.04080001, "close_rate": 0.0410045213283208, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.450979791426522, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 23:50:00+00:00", "close_date": "2018-01-24 03:35:00+00:00", "trade_duration": 225, "open_rate": 5.163e-05, "close_rate": 5.18887969924812e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1936.8584156498162, "profit_abs": 1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-24 00:20:00+00:00", "close_date": "2018-01-24 01:50:00+00:00", "trade_duration": 90, "open_rate": 0.04040781, "close_rate": 0.04061035541353383, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.474769110228938, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-24 06:45:00+00:00", "close_date": "2018-01-24 07:25:00+00:00", "trade_duration": 40, "open_rate": 5.132e-05, "close_rate": 5.157724310776942e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1948.5580670303975, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-24 14:15:00+00:00", "close_date": "2018-01-24 14:25:00+00:00", "trade_duration": 10, "open_rate": 5.198e-05, "close_rate": 5.432496240601503e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1923.8168526356292, "profit_abs": 0.0040000000000000036}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-24 14:50:00+00:00", "close_date": "2018-01-24 16:35:00+00:00", "trade_duration": 105, "open_rate": 3.054e-05, "close_rate": 3.069308270676692e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3274.3942370661425, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": 0.0, "open_date": "2018-01-24 15:10:00+00:00", "close_date": "2018-01-24 16:15:00+00:00", "trade_duration": 65, "open_rate": 9.263e-05, "close_rate": 9.309431077694236e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1079.5638562020945, "profit_abs": 2.7755575615628914e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-24 22:40:00+00:00", "close_date": "2018-01-24 23:25:00+00:00", "trade_duration": 45, "open_rate": 5.514e-05, "close_rate": 5.54163909774436e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1813.5654697134569, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-25 00:50:00+00:00", "close_date": "2018-01-25 01:30:00+00:00", "trade_duration": 40, "open_rate": 4.921e-05, "close_rate": 4.9456666666666664e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2032.1072952651903, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.0, "open_date": "2018-01-25 08:15:00+00:00", "close_date": "2018-01-25 12:15:00+00:00", "trade_duration": 240, "open_rate": 0.0026, "close_rate": 0.002613032581453634, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.46153846153847, "profit_abs": 1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 10:25:00+00:00", "close_date": "2018-01-25 16:15:00+00:00", "trade_duration": 350, "open_rate": 0.02799871, "close_rate": 0.028139054411027563, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.571593119825878, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 11:00:00+00:00", "close_date": "2018-01-25 11:45:00+00:00", "trade_duration": 45, "open_rate": 0.04078902, "close_rate": 0.0409934762406015, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4516401717913303, "profit_abs": -1.3877787807814457e-17}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 13:05:00+00:00", "close_date": "2018-01-25 13:45:00+00:00", "trade_duration": 40, "open_rate": 2.89e-05, "close_rate": 2.904486215538847e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3460.2076124567475, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 13:20:00+00:00", "close_date": "2018-01-25 14:05:00+00:00", "trade_duration": 45, "open_rate": 0.041103, "close_rate": 0.04130903007518797, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4329124394813033, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-25 15:45:00+00:00", "close_date": "2018-01-25 16:15:00+00:00", "trade_duration": 30, "open_rate": 5.428e-05, "close_rate": 5.509624060150376e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1842.2991893883568, "profit_abs": 0.0010000000000000148}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 17:45:00+00:00", "close_date": "2018-01-25 23:15:00+00:00", "trade_duration": 330, "open_rate": 5.414e-05, "close_rate": 5.441137844611528e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1847.063169560399, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 21:15:00+00:00", "close_date": "2018-01-25 21:55:00+00:00", "trade_duration": 40, "open_rate": 0.04140777, "close_rate": 0.0416153277443609, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.415005686130888, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.0, "open_date": "2018-01-26 02:05:00+00:00", "close_date": "2018-01-26 02:45:00+00:00", "trade_duration": 40, "open_rate": 0.00254309, "close_rate": 0.002555837318295739, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 39.32224183965177, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-26 02:55:00+00:00", "close_date": "2018-01-26 15:10:00+00:00", "trade_duration": 735, "open_rate": 5.607e-05, "close_rate": 5.6351052631578935e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1783.4849295523454, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.0, "open_date": "2018-01-26 06:10:00+00:00", "close_date": "2018-01-26 09:25:00+00:00", "trade_duration": 195, "open_rate": 0.00253806, "close_rate": 0.0025507821052631577, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 39.400171784748984, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-26 07:25:00+00:00", "close_date": "2018-01-26 09:55:00+00:00", "trade_duration": 150, "open_rate": 0.0415, "close_rate": 0.04170802005012531, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4096385542168677, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-26 09:55:00+00:00", "close_date": "2018-01-26 10:25:00+00:00", "trade_duration": 30, "open_rate": 5.321e-05, "close_rate": 5.401015037593984e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1879.3459875963165, "profit_abs": 0.000999999999999987}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-26 16:05:00+00:00", "close_date": "2018-01-26 16:45:00+00:00", "trade_duration": 40, "open_rate": 0.02772046, "close_rate": 0.02785940967418546, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.6074437437185387, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-26 23:35:00+00:00", "close_date": "2018-01-27 00:15:00+00:00", "trade_duration": 40, "open_rate": 0.09461341, "close_rate": 0.09508766268170424, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0569326272036914, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 00:35:00+00:00", "close_date": "2018-01-27 01:30:00+00:00", "trade_duration": 55, "open_rate": 5.615e-05, "close_rate": 5.643145363408521e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1780.9439002671415, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.07877175, "open_date": "2018-01-27 00:45:00+00:00", "close_date": "2018-01-30 04:45:00+00:00", "trade_duration": 4560, "open_rate": 5.556e-05, "close_rate": 5.144e-05, "open_at_end": true, "sell_reason": "force_sell", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1799.8560115190785, "profit_abs": -0.007896868250539965}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 02:30:00+00:00", "close_date": "2018-01-27 11:25:00+00:00", "trade_duration": 535, "open_rate": 0.06900001, "close_rate": 0.06934587471177944, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4492751522789635, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 06:25:00+00:00", "close_date": "2018-01-27 07:05:00+00:00", "trade_duration": 40, "open_rate": 0.09449985, "close_rate": 0.0949735334586466, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.058202737887944, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.04815133, "open_date": "2018-01-27 09:40:00+00:00", "close_date": "2018-01-30 04:40:00+00:00", "trade_duration": 4020, "open_rate": 0.0410697, "close_rate": 0.03928809, "open_at_end": true, "sell_reason": "force_sell", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4348850855983852, "profit_abs": -0.004827170578309559}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 11:45:00+00:00", "close_date": "2018-01-27 12:30:00+00:00", "trade_duration": 45, "open_rate": 0.0285, "close_rate": 0.02864285714285714, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.5087719298245617, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 12:35:00+00:00", "close_date": "2018-01-27 15:25:00+00:00", "trade_duration": 170, "open_rate": 0.02866372, "close_rate": 0.02880739779448621, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.4887307020861216, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 15:50:00+00:00", "close_date": "2018-01-27 16:50:00+00:00", "trade_duration": 60, "open_rate": 0.095381, "close_rate": 0.09585910025062656, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0484268355332824, "profit_abs": 1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 17:05:00+00:00", "close_date": "2018-01-27 17:45:00+00:00", "trade_duration": 40, "open_rate": 0.06759092, "close_rate": 0.06792972160401002, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4794886650455417, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 23:40:00+00:00", "close_date": "2018-01-28 01:05:00+00:00", "trade_duration": 85, "open_rate": 0.00258501, "close_rate": 0.002597967443609022, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.684569885609726, "profit_abs": -1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-28 02:25:00+00:00", "close_date": "2018-01-28 08:10:00+00:00", "trade_duration": 345, "open_rate": 0.06698502, "close_rate": 0.0673207845112782, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4928710926711672, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-28 10:25:00+00:00", "close_date": "2018-01-28 16:30:00+00:00", "trade_duration": 365, "open_rate": 0.0677177, "close_rate": 0.06805713709273183, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4767187899175547, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-28 20:35:00+00:00", "close_date": "2018-01-28 21:35:00+00:00", "trade_duration": 60, "open_rate": 5.215e-05, "close_rate": 5.2411403508771925e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1917.5455417066157, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-28 22:00:00+00:00", "close_date": "2018-01-28 22:30:00+00:00", "trade_duration": 30, "open_rate": 0.00273809, "close_rate": 0.002779264285714285, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 36.5218089982433, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-29 00:00:00+00:00", "close_date": "2018-01-29 00:30:00+00:00", "trade_duration": 30, "open_rate": 0.00274632, "close_rate": 0.002787618045112782, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 36.412362725392526, "profit_abs": 0.0010000000000000148}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-29 02:15:00+00:00", "close_date": "2018-01-29 03:00:00+00:00", "trade_duration": 45, "open_rate": 0.01622478, "close_rate": 0.016306107218045113, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.163411768911504, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 03:05:00+00:00", "close_date": "2018-01-29 03:45:00+00:00", "trade_duration": 40, "open_rate": 0.069, "close_rate": 0.06934586466165413, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4492753623188406, "profit_abs": -1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 05:20:00+00:00", "close_date": "2018-01-29 06:55:00+00:00", "trade_duration": 95, "open_rate": 8.755e-05, "close_rate": 8.798884711779448e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1142.204454597373, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 07:00:00+00:00", "close_date": "2018-01-29 19:25:00+00:00", "trade_duration": 745, "open_rate": 0.06825763, "close_rate": 0.06859977350877192, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4650376815016872, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 19:45:00+00:00", "close_date": "2018-01-29 20:25:00+00:00", "trade_duration": 40, "open_rate": 0.06713892, "close_rate": 0.06747545593984962, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4894490408841845, "profit_abs": -1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": -0.0199116, "open_date": "2018-01-29 23:30:00+00:00", "close_date": "2018-01-30 04:45:00+00:00", "trade_duration": 315, "open_rate": 8.934e-05, "close_rate": 8.8e-05, "open_at_end": true, "sell_reason": "force_sell", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1119.3194537721067, "profit_abs": -0.0019961383478844796}], "results_per_pair": [{"key": "TRX/BTC", "trades": 15, "profit_mean": 0.0023467073333333323, "profit_mean_pct": 0.23467073333333321, "profit_sum": 0.035200609999999986, "profit_sum_pct": 3.5200609999999988, "profit_total_abs": 0.0035288616521155086, "profit_total_pct": 1.1733536666666662, "duration_avg": "2:28:00", "wins": 9, "draws": 2, "losses": 4}, {"key": "ADA/BTC", "trades": 29, "profit_mean": -0.0011598141379310352, "profit_mean_pct": -0.11598141379310352, "profit_sum": -0.03363461000000002, "profit_sum_pct": -3.3634610000000023, "profit_total_abs": -0.0033718682505400333, "profit_total_pct": -1.1211536666666675, "duration_avg": "5:35:00", "wins": 9, "draws": 11, "losses": 9}, {"key": "XLM/BTC", "trades": 21, "profit_mean": 0.0026243899999999994, "profit_mean_pct": 0.2624389999999999, "profit_sum": 0.05511218999999999, "profit_sum_pct": 5.511218999999999, "profit_total_abs": 0.005525000000000002, "profit_total_pct": 1.8370729999999995, "duration_avg": "3:21:00", "wins": 12, "draws": 3, "losses": 6}, {"key": "ETH/BTC", "trades": 21, "profit_mean": 0.0009500057142857142, "profit_mean_pct": 0.09500057142857142, "profit_sum": 0.01995012, "profit_sum_pct": 1.9950119999999998, "profit_total_abs": 0.0019999999999999463, "profit_total_pct": 0.6650039999999999, "duration_avg": "2:17:00", "wins": 5, "draws": 10, "losses": 6}, {"key": "XMR/BTC", "trades": 16, "profit_mean": -0.0027899012500000007, "profit_mean_pct": -0.2789901250000001, "profit_sum": -0.04463842000000001, "profit_sum_pct": -4.463842000000001, "profit_total_abs": -0.0044750000000000345, "profit_total_pct": -1.4879473333333337, "duration_avg": "8:41:00", "wins": 6, "draws": 5, "losses": 5}, {"key": "ZEC/BTC", "trades": 21, "profit_mean": -0.00039290904761904774, "profit_mean_pct": -0.03929090476190478, "profit_sum": -0.008251090000000003, "profit_sum_pct": -0.8251090000000003, "profit_total_abs": -0.000827170578309569, "profit_total_pct": -0.27503633333333344, "duration_avg": "4:17:00", "wins": 8, "draws": 7, "losses": 6}, {"key": "NXT/BTC", "trades": 12, "profit_mean": -0.0012261025000000006, "profit_mean_pct": -0.12261025000000006, "profit_sum": -0.014713230000000008, "profit_sum_pct": -1.4713230000000008, "profit_total_abs": -0.0014750000000000874, "profit_total_pct": -0.4904410000000003, "duration_avg": "0:57:00", "wins": 4, "draws": 3, "losses": 5}, {"key": "LTC/BTC", "trades": 8, "profit_mean": 0.00748129625, "profit_mean_pct": 0.748129625, "profit_sum": 0.05985037, "profit_sum_pct": 5.985037, "profit_total_abs": 0.006000000000000019, "profit_total_pct": 1.9950123333333334, "duration_avg": "1:59:00", "wins": 5, "draws": 2, "losses": 1}, {"key": "ETC/BTC", "trades": 20, "profit_mean": 0.0022568569999999997, "profit_mean_pct": 0.22568569999999996, "profit_sum": 0.04513713999999999, "profit_sum_pct": 4.513713999999999, "profit_total_abs": 0.004525000000000001, "profit_total_pct": 1.504571333333333, "duration_avg": "1:45:00", "wins": 11, "draws": 4, "losses": 5}, {"key": "DASH/BTC", "trades": 16, "profit_mean": 0.0018703237499999997, "profit_mean_pct": 0.18703237499999997, "profit_sum": 0.029925179999999996, "profit_sum_pct": 2.9925179999999996, "profit_total_abs": 0.002999999999999961, "profit_total_pct": 0.9975059999999999, "duration_avg": "3:03:00", "wins": 4, "draws": 7, "losses": 5}, {"key": "TOTAL", "trades": 179, "profit_mean": 0.0008041243575418989, "profit_mean_pct": 0.0804124357541899, "profit_sum": 0.1439382599999999, "profit_sum_pct": 14.39382599999999, "profit_total_abs": 0.014429822823265714, "profit_total_pct": 4.797941999999996, "duration_avg": "3:40:00", "wins": 73, "draws": 54, "losses": 52}], "sell_reason_summary": [{"sell_reason": "roi", "trades": 170, "wins": 73, "draws": 54, "losses": 43, "profit_mean": 0.005398268352941177, "profit_mean_pct": 0.54, "profit_sum": 0.91770562, "profit_sum_pct": 91.77, "profit_total_abs": 0.09199999999999964, "profit_pct_total": 30.59}, {"sell_reason": "stop_loss", "trades": 6, "wins": 0, "draws": 0, "losses": 6, "profit_mean": -0.10448878000000002, "profit_mean_pct": -10.45, "profit_sum": -0.6269326800000001, "profit_sum_pct": -62.69, "profit_total_abs": -0.06284999999999992, "profit_pct_total": -20.9}, {"sell_reason": "force_sell", "trades": 3, "wins": 0, "draws": 0, "losses": 3, "profit_mean": -0.04894489333333333, "profit_mean_pct": -4.89, "profit_sum": -0.14683468, "profit_sum_pct": -14.68, "profit_total_abs": -0.014720177176734003, "profit_pct_total": -4.89}], "left_open_trades": [{"key": "TRX/BTC", "trades": 1, "profit_mean": -0.0199116, "profit_mean_pct": -1.9911600000000003, "profit_sum": -0.0199116, "profit_sum_pct": -1.9911600000000003, "profit_total_abs": -0.0019961383478844796, "profit_total_pct": -0.6637200000000001, "duration_avg": "5:15:00", "wins": 0, "draws": 0, "losses": 1}, {"key": "ADA/BTC", "trades": 1, "profit_mean": -0.07877175, "profit_mean_pct": -7.877175, "profit_sum": -0.07877175, "profit_sum_pct": -7.877175, "profit_total_abs": -0.007896868250539965, "profit_total_pct": -2.625725, "duration_avg": "3 days, 4:00:00", "wins": 0, "draws": 0, "losses": 1}, {"key": "ZEC/BTC", "trades": 1, "profit_mean": -0.04815133, "profit_mean_pct": -4.815133, "profit_sum": -0.04815133, "profit_sum_pct": -4.815133, "profit_total_abs": -0.004827170578309559, "profit_total_pct": -1.6050443333333335, "duration_avg": "2 days, 19:00:00", "wins": 0, "draws": 0, "losses": 1}, {"key": "TOTAL", "trades": 3, "profit_mean": -0.04894489333333333, "profit_mean_pct": -4.894489333333333, "profit_sum": -0.14683468, "profit_sum_pct": -14.683468, "profit_total_abs": -0.014720177176734003, "profit_total_pct": -4.8944893333333335, "duration_avg": "2 days, 1:25:00", "wins": 0, "draws": 0, "losses": 3}], "total_trades": 179, "backtest_start": "2018-01-30 04:45:00+00:00", "backtest_start_ts": 1517287500, "backtest_end": "2018-01-30 04:45:00+00:00", "backtest_end_ts": 1517287500, "backtest_days": 0, "trades_per_day": null, "market_change": 0.25, "stake_amount": 0.1, "max_drawdown": 0.21142322000000008, "drawdown_start": "2018-01-24 14:25:00+00:00", "drawdown_start_ts": 1516803900.0, "drawdown_end": "2018-01-30 04:45:00+00:00", "drawdown_end_ts": 1517287500.0, "pairlist": ["TRX/BTC", "ADA/BTC", "XLM/BTC", "ETH/BTC", "XMR/BTC", "ZEC/BTC","NXT/BTC", "LTC/BTC", "ETC/BTC", "DASH/BTC"]}, "TestStrategy": {"trades": [{"pair": "TRX/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:15:00+00:00", "close_date": "2018-01-10 07:20:00+00:00", "trade_duration": 5, "open_rate": 9.64e-05, "close_rate": 0.00010074887218045112, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1037.344398340249, "profit_abs": 0.00399999999999999}, {"pair": "ADA/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:15:00+00:00", "close_date": "2018-01-10 07:30:00+00:00", "trade_duration": 15, "open_rate": 4.756e-05, "close_rate": 4.9705563909774425e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2102.6072329688814, "profit_abs": 0.00399999999999999}, {"pair": "XLM/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:25:00+00:00", "close_date": "2018-01-10 07:35:00+00:00", "trade_duration": 10, "open_rate": 3.339e-05, "close_rate": 3.489631578947368e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2994.908655286014, "profit_abs": 0.0040000000000000036}, {"pair": "TRX/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:25:00+00:00", "close_date": "2018-01-10 07:40:00+00:00", "trade_duration": 15, "open_rate": 9.696e-05, "close_rate": 0.00010133413533834584, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1031.3531353135315, "profit_abs": 0.00399999999999999}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 07:35:00+00:00", "close_date": "2018-01-10 08:35:00+00:00", "trade_duration": 60, "open_rate": 0.0943, "close_rate": 0.09477268170426063, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0604453870625663, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-10 07:40:00+00:00", "close_date": "2018-01-10 08:10:00+00:00", "trade_duration": 30, "open_rate": 0.02719607, "close_rate": 0.02760503345864661, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.677001860930642, "profit_abs": 0.0010000000000000009}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-10 08:15:00+00:00", "close_date": "2018-01-10 09:55:00+00:00", "trade_duration": 100, "open_rate": 0.04634952, "close_rate": 0.046581848421052625, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.1575196463739, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 14:45:00+00:00", "close_date": "2018-01-10 15:50:00+00:00", "trade_duration": 65, "open_rate": 3.066e-05, "close_rate": 3.081368421052631e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3261.5786040443577, "profit_abs": -1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-10 16:35:00+00:00", "close_date": "2018-01-10 17:15:00+00:00", "trade_duration": 40, "open_rate": 0.0168999, "close_rate": 0.016984611278195488, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 5.917194776300452, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 16:40:00+00:00", "close_date": "2018-01-10 17:20:00+00:00", "trade_duration": 40, "open_rate": 0.09132568, "close_rate": 0.0917834528320802, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0949822656672252, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 18:50:00+00:00", "close_date": "2018-01-10 19:45:00+00:00", "trade_duration": 55, "open_rate": 0.08898003, "close_rate": 0.08942604518796991, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1238476768326557, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-10 22:15:00+00:00", "close_date": "2018-01-10 23:00:00+00:00", "trade_duration": 45, "open_rate": 0.08560008, "close_rate": 0.08602915308270676, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1682232072680307, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-10 22:50:00+00:00", "close_date": "2018-01-10 23:20:00+00:00", "trade_duration": 30, "open_rate": 0.00249083, "close_rate": 0.0025282860902255634, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 40.147260150231055, "profit_abs": 0.000999999999999987}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 23:15:00+00:00", "close_date": "2018-01-11 00:15:00+00:00", "trade_duration": 60, "open_rate": 3.022e-05, "close_rate": 3.037147869674185e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3309.0668431502318, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-10 23:40:00+00:00", "close_date": "2018-01-11 00:05:00+00:00", "trade_duration": 25, "open_rate": 0.002437, "close_rate": 0.0024980776942355883, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 41.03405826836274, "profit_abs": 0.001999999999999974}, {"pair": "ZEC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 00:00:00+00:00", "close_date": "2018-01-11 00:35:00+00:00", "trade_duration": 35, "open_rate": 0.04771803, "close_rate": 0.04843559436090225, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.0956439316543456, "profit_abs": 0.0010000000000000009}, {"pair": "XLM/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-11 03:40:00+00:00", "close_date": "2018-01-11 04:25:00+00:00", "trade_duration": 45, "open_rate": 3.651e-05, "close_rate": 3.2859000000000005e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2738.9756231169545, "profit_abs": -0.01047499999999997}, {"pair": "ETH/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 03:55:00+00:00", "close_date": "2018-01-11 04:25:00+00:00", "trade_duration": 30, "open_rate": 0.08824105, "close_rate": 0.08956798308270676, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1332594070446804, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 04:00:00+00:00", "close_date": "2018-01-11 04:50:00+00:00", "trade_duration": 50, "open_rate": 0.00243, "close_rate": 0.002442180451127819, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 41.1522633744856, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 04:30:00+00:00", "close_date": "2018-01-11 04:55:00+00:00", "trade_duration": 25, "open_rate": 0.04545064, "close_rate": 0.046589753784461146, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.200189040242338, "profit_abs": 0.001999999999999988}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 04:30:00+00:00", "close_date": "2018-01-11 04:50:00+00:00", "trade_duration": 20, "open_rate": 3.372e-05, "close_rate": 3.456511278195488e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2965.599051008304, "profit_abs": 0.001999999999999988}, {"pair": "XMR/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 04:55:00+00:00", "close_date": "2018-01-11 05:15:00+00:00", "trade_duration": 20, "open_rate": 0.02644, "close_rate": 0.02710265664160401, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.7821482602118004, "profit_abs": 0.001999999999999988}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 11:20:00+00:00", "close_date": "2018-01-11 12:00:00+00:00", "trade_duration": 40, "open_rate": 0.08812, "close_rate": 0.08856170426065162, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1348161597821154, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 11:35:00+00:00", "close_date": "2018-01-11 12:15:00+00:00", "trade_duration": 40, "open_rate": 0.02683577, "close_rate": 0.026970285137844607, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.7263696923919087, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 14:00:00+00:00", "close_date": "2018-01-11 14:25:00+00:00", "trade_duration": 25, "open_rate": 4.919e-05, "close_rate": 5.04228320802005e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2032.9335230737956, "profit_abs": 0.0020000000000000018}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 19:25:00+00:00", "close_date": "2018-01-11 20:35:00+00:00", "trade_duration": 70, "open_rate": 0.08784896, "close_rate": 0.08828930566416039, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1383174029607181, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 22:35:00+00:00", "close_date": "2018-01-11 23:30:00+00:00", "trade_duration": 55, "open_rate": 5.105e-05, "close_rate": 5.130588972431077e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1958.8638589618022, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 22:55:00+00:00", "close_date": "2018-01-11 23:25:00+00:00", "trade_duration": 30, "open_rate": 3.96e-05, "close_rate": 4.019548872180451e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2525.252525252525, "profit_abs": 0.0010000000000000148}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 22:55:00+00:00", "close_date": "2018-01-11 23:35:00+00:00", "trade_duration": 40, "open_rate": 2.885e-05, "close_rate": 2.899461152882205e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3466.204506065858, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 23:30:00+00:00", "close_date": "2018-01-12 00:05:00+00:00", "trade_duration": 35, "open_rate": 0.02645, "close_rate": 0.026847744360902256, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.780718336483932, "profit_abs": 0.0010000000000000148}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 23:55:00+00:00", "close_date": "2018-01-12 01:15:00+00:00", "trade_duration": 80, "open_rate": 0.048, "close_rate": 0.04824060150375939, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.0833333333333335, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-12 21:15:00+00:00", "close_date": "2018-01-12 21:40:00+00:00", "trade_duration": 25, "open_rate": 4.692e-05, "close_rate": 4.809593984962405e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2131.287297527707, "profit_abs": 0.001999999999999974}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 00:55:00+00:00", "close_date": "2018-01-13 06:20:00+00:00", "trade_duration": 325, "open_rate": 0.00256966, "close_rate": 0.0025825405012531327, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.91565421106294, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.0, "open_date": "2018-01-13 10:55:00+00:00", "close_date": "2018-01-13 11:35:00+00:00", "trade_duration": 40, "open_rate": 6.262e-05, "close_rate": 6.293388471177944e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1596.933886937081, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-13 13:05:00+00:00", "close_date": "2018-01-15 14:10:00+00:00", "trade_duration": 2945, "open_rate": 4.73e-05, "close_rate": 4.753709273182957e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2114.1649048625795, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 13:30:00+00:00", "close_date": "2018-01-13 14:45:00+00:00", "trade_duration": 75, "open_rate": 6.063e-05, "close_rate": 6.0933909774436085e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1649.348507339601, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 13:40:00+00:00", "close_date": "2018-01-13 23:30:00+00:00", "trade_duration": 590, "open_rate": 0.00011082, "close_rate": 0.00011137548872180448, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 902.3641941887746, "profit_abs": -2.7755575615628914e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 15:15:00+00:00", "close_date": "2018-01-13 15:55:00+00:00", "trade_duration": 40, "open_rate": 5.93e-05, "close_rate": 5.9597243107769415e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1686.3406408094436, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 16:30:00+00:00", "close_date": "2018-01-13 17:10:00+00:00", "trade_duration": 40, "open_rate": 0.04850003, "close_rate": 0.04874313791979949, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.0618543947292407, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 22:05:00+00:00", "close_date": "2018-01-14 06:25:00+00:00", "trade_duration": 500, "open_rate": 0.09825019, "close_rate": 0.09874267215538848, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0178097365511456, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": 0.0, "open_date": "2018-01-14 00:20:00+00:00", "close_date": "2018-01-14 22:55:00+00:00", "trade_duration": 1355, "open_rate": 6.018e-05, "close_rate": 6.048165413533834e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1661.681621801263, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-14 12:45:00+00:00", "close_date": "2018-01-14 13:25:00+00:00", "trade_duration": 40, "open_rate": 0.09758999, "close_rate": 0.0980791628822055, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.024695258191952, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-14 15:30:00+00:00", "close_date": "2018-01-14 16:00:00+00:00", "trade_duration": 30, "open_rate": 0.00311, "close_rate": 0.0031567669172932328, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 32.154340836012864, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-14 20:45:00+00:00", "close_date": "2018-01-14 22:15:00+00:00", "trade_duration": 90, "open_rate": 0.00312401, "close_rate": 0.003139669197994987, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 32.010140812609436, "profit_abs": -1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-14 23:35:00+00:00", "close_date": "2018-01-15 00:30:00+00:00", "trade_duration": 55, "open_rate": 0.0174679, "close_rate": 0.017555458395989976, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 5.724786608579165, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-14 23:45:00+00:00", "close_date": "2018-01-15 00:25:00+00:00", "trade_duration": 40, "open_rate": 0.07346846, "close_rate": 0.07383672295739348, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.3611282991367997, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 02:25:00+00:00", "close_date": "2018-01-15 03:05:00+00:00", "trade_duration": 40, "open_rate": 0.097994, "close_rate": 0.09848519799498744, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.020470641059657, "profit_abs": -2.7755575615628914e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 07:20:00+00:00", "close_date": "2018-01-15 08:00:00+00:00", "trade_duration": 40, "open_rate": 0.09659, "close_rate": 0.09707416040100247, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0353038616834043, "profit_abs": -2.7755575615628914e-17}, {"pair": "TRX/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-15 08:20:00+00:00", "close_date": "2018-01-15 08:55:00+00:00", "trade_duration": 35, "open_rate": 9.987e-05, "close_rate": 0.00010137180451127818, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1001.3016921998599, "profit_abs": 0.0010000000000000009}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-15 12:10:00+00:00", "close_date": "2018-01-16 02:50:00+00:00", "trade_duration": 880, "open_rate": 0.0948969, "close_rate": 0.09537257368421052, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0537752023511833, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 14:10:00+00:00", "close_date": "2018-01-15 17:40:00+00:00", "trade_duration": 210, "open_rate": 0.071, "close_rate": 0.07135588972431077, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4084507042253522, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 14:30:00+00:00", "close_date": "2018-01-15 15:10:00+00:00", "trade_duration": 40, "open_rate": 0.04600501, "close_rate": 0.046235611553884705, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.173676301776698, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 18:10:00+00:00", "close_date": "2018-01-15 19:25:00+00:00", "trade_duration": 75, "open_rate": 9.438e-05, "close_rate": 9.485308270676693e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1059.5465140919687, "profit_abs": 1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 18:35:00+00:00", "close_date": "2018-01-15 19:15:00+00:00", "trade_duration": 40, "open_rate": 0.03040001, "close_rate": 0.030552391002506264, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.2894726021471703, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-15 20:25:00+00:00", "close_date": "2018-01-16 08:25:00+00:00", "trade_duration": 720, "open_rate": 5.837e-05, "close_rate": 5.2533e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1713.2088401576154, "profit_abs": -0.010474999999999984}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 20:40:00+00:00", "close_date": "2018-01-15 22:00:00+00:00", "trade_duration": 80, "open_rate": 0.046036, "close_rate": 0.04626675689223057, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.1722130506560084, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-16 00:30:00+00:00", "close_date": "2018-01-16 01:10:00+00:00", "trade_duration": 40, "open_rate": 0.0028685, "close_rate": 0.0028828784461152877, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 34.86142583231654, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": 0.0, "open_date": "2018-01-16 01:15:00+00:00", "close_date": "2018-01-16 02:35:00+00:00", "trade_duration": 80, "open_rate": 0.06731755, "close_rate": 0.0676549813283208, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4854967241083492, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-16 07:45:00+00:00", "close_date": "2018-01-16 08:40:00+00:00", "trade_duration": 55, "open_rate": 0.09217614, "close_rate": 0.09263817578947368, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0848794492804754, "profit_abs": 0.0}, {"pair": "LTC/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 08:35:00+00:00", "close_date": "2018-01-16 08:55:00+00:00", "trade_duration": 20, "open_rate": 0.0165, "close_rate": 0.016913533834586467, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.0606060606060606, "profit_abs": 0.0020000000000000018}, {"pair": "TRX/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 08:35:00+00:00", "close_date": "2018-01-16 08:40:00+00:00", "trade_duration": 5, "open_rate": 7.953e-05, "close_rate": 8.311781954887218e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1257.387149503332, "profit_abs": 0.00399999999999999}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-16 08:45:00+00:00", "close_date": "2018-01-16 09:50:00+00:00", "trade_duration": 65, "open_rate": 0.045202, "close_rate": 0.04542857644110275, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.2122914915269236, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 09:15:00+00:00", "close_date": "2018-01-16 09:45:00+00:00", "trade_duration": 30, "open_rate": 5.248e-05, "close_rate": 5.326917293233082e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1905.487804878049, "profit_abs": 0.0010000000000000009}, {"pair": "XMR/BTC", "profit_percent": 0.0, "open_date": "2018-01-16 09:15:00+00:00", "close_date": "2018-01-16 09:55:00+00:00", "trade_duration": 40, "open_rate": 0.02892318, "close_rate": 0.02906815834586466, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.457434486802627, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 09:50:00+00:00", "close_date": "2018-01-16 10:10:00+00:00", "trade_duration": 20, "open_rate": 5.158e-05, "close_rate": 5.287273182957392e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1938.735944164405, "profit_abs": 0.001999999999999988}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 10:05:00+00:00", "close_date": "2018-01-16 10:35:00+00:00", "trade_duration": 30, "open_rate": 0.02828232, "close_rate": 0.02870761804511278, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.5357778286929786, "profit_abs": 0.0010000000000000009}, {"pair": "ZEC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 10:05:00+00:00", "close_date": "2018-01-16 10:40:00+00:00", "trade_duration": 35, "open_rate": 0.04357584, "close_rate": 0.044231115789473675, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.294849623093898, "profit_abs": 0.0010000000000000009}, {"pair": "ADA/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 13:45:00+00:00", "close_date": "2018-01-16 14:20:00+00:00", "trade_duration": 35, "open_rate": 5.362e-05, "close_rate": 5.442631578947368e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1864.975755315181, "profit_abs": 0.0010000000000000148}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-16 17:30:00+00:00", "close_date": "2018-01-16 18:25:00+00:00", "trade_duration": 55, "open_rate": 5.302e-05, "close_rate": 5.328576441102756e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1886.0807242549984, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 18:15:00+00:00", "close_date": "2018-01-16 18:45:00+00:00", "trade_duration": 30, "open_rate": 0.09129999, "close_rate": 0.09267292218045112, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0952903718828448, "profit_abs": 0.0010000000000000148}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 18:15:00+00:00", "close_date": "2018-01-16 18:35:00+00:00", "trade_duration": 20, "open_rate": 3.808e-05, "close_rate": 3.903438596491228e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2626.0504201680674, "profit_abs": 0.0020000000000000018}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 19:00:00+00:00", "close_date": "2018-01-16 19:30:00+00:00", "trade_duration": 30, "open_rate": 0.02811012, "close_rate": 0.028532828571428567, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.557437677249333, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-16 21:25:00+00:00", "close_date": "2018-01-16 22:25:00+00:00", "trade_duration": 60, "open_rate": 0.00258379, "close_rate": 0.002325411, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.702835756775904, "profit_abs": -0.010474999999999984}, {"pair": "NXT/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-16 21:25:00+00:00", "close_date": "2018-01-16 22:45:00+00:00", "trade_duration": 80, "open_rate": 2.559e-05, "close_rate": 2.3031e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3907.7764751856193, "profit_abs": -0.010474999999999998}, {"pair": "TRX/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-16 21:35:00+00:00", "close_date": "2018-01-16 22:25:00+00:00", "trade_duration": 50, "open_rate": 7.62e-05, "close_rate": 6.858e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1312.3359580052495, "profit_abs": -0.010474999999999984}, {"pair": "ETC/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:30:00+00:00", "close_date": "2018-01-16 22:35:00+00:00", "trade_duration": 5, "open_rate": 0.00229844, "close_rate": 0.002402129022556391, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 43.507770487809125, "profit_abs": 0.004000000000000017}, {"pair": "LTC/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:30:00+00:00", "close_date": "2018-01-16 22:40:00+00:00", "trade_duration": 10, "open_rate": 0.0151, "close_rate": 0.015781203007518795, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.622516556291391, "profit_abs": 0.00399999999999999}, {"pair": "ETC/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:40:00+00:00", "close_date": "2018-01-16 22:45:00+00:00", "trade_duration": 5, "open_rate": 0.00235676, "close_rate": 0.00246308, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 42.431134269081284, "profit_abs": 0.0040000000000000036}, {"pair": "DASH/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 22:45:00+00:00", "close_date": "2018-01-16 23:05:00+00:00", "trade_duration": 20, "open_rate": 0.0630692, "close_rate": 0.06464988170426066, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.585559988076589, "profit_abs": 0.0020000000000000018}, {"pair": "NXT/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:50:00+00:00", "close_date": "2018-01-16 22:55:00+00:00", "trade_duration": 5, "open_rate": 2.2e-05, "close_rate": 2.299248120300751e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 4545.454545454546, "profit_abs": 0.003999999999999976}, {"pair": "ADA/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-17 03:30:00+00:00", "close_date": "2018-01-17 04:00:00+00:00", "trade_duration": 30, "open_rate": 4.974e-05, "close_rate": 5.048796992481203e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2010.454362685967, "profit_abs": 0.0010000000000000009}, {"pair": "TRX/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-17 03:55:00+00:00", "close_date": "2018-01-17 04:15:00+00:00", "trade_duration": 20, "open_rate": 7.108e-05, "close_rate": 7.28614536340852e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1406.8655036578502, "profit_abs": 0.001999999999999974}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 09:35:00+00:00", "close_date": "2018-01-17 10:15:00+00:00", "trade_duration": 40, "open_rate": 0.04327, "close_rate": 0.04348689223057644, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.3110700254217704, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 10:20:00+00:00", "close_date": "2018-01-17 17:00:00+00:00", "trade_duration": 400, "open_rate": 4.997e-05, "close_rate": 5.022047619047618e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2001.2007204322595, "profit_abs": -1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 10:30:00+00:00", "close_date": "2018-01-17 11:25:00+00:00", "trade_duration": 55, "open_rate": 0.06836818, "close_rate": 0.06871087764411027, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4626687444363737, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 10:30:00+00:00", "close_date": "2018-01-17 11:10:00+00:00", "trade_duration": 40, "open_rate": 3.63e-05, "close_rate": 3.648195488721804e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2754.8209366391184, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 12:30:00+00:00", "close_date": "2018-01-17 22:05:00+00:00", "trade_duration": 575, "open_rate": 0.0281, "close_rate": 0.02824085213032581, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.5587188612099645, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 12:35:00+00:00", "close_date": "2018-01-17 16:55:00+00:00", "trade_duration": 260, "open_rate": 0.08651001, "close_rate": 0.08694364413533832, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1559355963546878, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 05:00:00+00:00", "close_date": "2018-01-18 05:55:00+00:00", "trade_duration": 55, "open_rate": 5.633e-05, "close_rate": 5.6612355889724306e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1775.2529735487308, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-18 05:20:00+00:00", "close_date": "2018-01-18 05:55:00+00:00", "trade_duration": 35, "open_rate": 0.06988494, "close_rate": 0.07093584135338346, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.430923457900944, "profit_abs": 0.0010000000000000009}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 07:35:00+00:00", "close_date": "2018-01-18 08:15:00+00:00", "trade_duration": 40, "open_rate": 5.545e-05, "close_rate": 5.572794486215538e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1803.4265103697026, "profit_abs": -1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 09:00:00+00:00", "close_date": "2018-01-18 09:40:00+00:00", "trade_duration": 40, "open_rate": 0.01633527, "close_rate": 0.016417151052631574, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.121723118136401, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 16:40:00+00:00", "close_date": "2018-01-18 17:20:00+00:00", "trade_duration": 40, "open_rate": 0.00269734, "close_rate": 0.002710860501253133, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 37.073561360451414, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-18 18:05:00+00:00", "close_date": "2018-01-18 18:30:00+00:00", "trade_duration": 25, "open_rate": 4.475e-05, "close_rate": 4.587155388471177e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2234.63687150838, "profit_abs": 0.0020000000000000018}, {"pair": "NXT/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-18 18:25:00+00:00", "close_date": "2018-01-18 18:55:00+00:00", "trade_duration": 30, "open_rate": 2.79e-05, "close_rate": 2.8319548872180444e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3584.2293906810037, "profit_abs": 0.000999999999999987}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 20:10:00+00:00", "close_date": "2018-01-18 20:50:00+00:00", "trade_duration": 40, "open_rate": 0.04439326, "close_rate": 0.04461578260651629, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.2525942001105577, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 21:30:00+00:00", "close_date": "2018-01-19 00:35:00+00:00", "trade_duration": 185, "open_rate": 4.49e-05, "close_rate": 4.51250626566416e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2227.1714922049, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 21:55:00+00:00", "close_date": "2018-01-19 05:05:00+00:00", "trade_duration": 430, "open_rate": 0.02855, "close_rate": 0.028693107769423555, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.502626970227671, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 22:10:00+00:00", "close_date": "2018-01-18 22:50:00+00:00", "trade_duration": 40, "open_rate": 5.796e-05, "close_rate": 5.8250526315789473e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1725.3278122843342, "profit_abs": 1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 23:50:00+00:00", "close_date": "2018-01-19 00:30:00+00:00", "trade_duration": 40, "open_rate": 0.04340323, "close_rate": 0.04362079005012531, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.303975994413319, "profit_abs": 1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-19 16:45:00+00:00", "close_date": "2018-01-19 17:35:00+00:00", "trade_duration": 50, "open_rate": 0.04454455, "close_rate": 0.04476783095238095, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.244943545282195, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-19 17:15:00+00:00", "close_date": "2018-01-19 19:55:00+00:00", "trade_duration": 160, "open_rate": 5.62e-05, "close_rate": 5.648170426065162e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1779.3594306049824, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-19 17:20:00+00:00", "close_date": "2018-01-19 20:15:00+00:00", "trade_duration": 175, "open_rate": 4.339e-05, "close_rate": 4.360749373433584e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2304.6784973496196, "profit_abs": -1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": 0.0, "open_date": "2018-01-20 04:45:00+00:00", "close_date": "2018-01-20 17:35:00+00:00", "trade_duration": 770, "open_rate": 0.0001009, "close_rate": 0.00010140576441102755, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 991.0802775024778, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 04:50:00+00:00", "close_date": "2018-01-20 15:15:00+00:00", "trade_duration": 625, "open_rate": 0.00270505, "close_rate": 0.002718609147869674, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 36.96789338459548, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 04:50:00+00:00", "close_date": "2018-01-20 07:00:00+00:00", "trade_duration": 130, "open_rate": 0.03000002, "close_rate": 0.030150396040100245, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.3333311111125927, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 09:00:00+00:00", "close_date": "2018-01-20 09:40:00+00:00", "trade_duration": 40, "open_rate": 5.46e-05, "close_rate": 5.4873684210526304e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1831.5018315018317, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-20 18:25:00+00:00", "close_date": "2018-01-25 03:50:00+00:00", "trade_duration": 6325, "open_rate": 0.03082222, "close_rate": 0.027739998, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.244412634781012, "profit_abs": -0.010474999999999998}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 22:25:00+00:00", "close_date": "2018-01-20 23:15:00+00:00", "trade_duration": 50, "open_rate": 0.08969999, "close_rate": 0.09014961401002504, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1148273260677064, "profit_abs": 0.0}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-21 02:50:00+00:00", "close_date": "2018-01-21 14:30:00+00:00", "trade_duration": 700, "open_rate": 0.01632501, "close_rate": 0.01640683962406015, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.125570520324337, "profit_abs": 1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-21 10:20:00+00:00", "close_date": "2018-01-21 11:00:00+00:00", "trade_duration": 40, "open_rate": 0.070538, "close_rate": 0.07089157393483708, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.417675579120474, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-21 15:50:00+00:00", "close_date": "2018-01-21 18:45:00+00:00", "trade_duration": 175, "open_rate": 5.301e-05, "close_rate": 5.327571428571427e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1886.4365214110546, "profit_abs": -2.7755575615628914e-17}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-21 16:20:00+00:00", "close_date": "2018-01-21 17:00:00+00:00", "trade_duration": 40, "open_rate": 3.955e-05, "close_rate": 3.9748245614035085e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2528.4450063211125, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-21 21:15:00+00:00", "close_date": "2018-01-21 21:45:00+00:00", "trade_duration": 30, "open_rate": 0.00258505, "close_rate": 0.002623922932330827, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.6839712964933, "profit_abs": 0.0010000000000000009}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-21 21:15:00+00:00", "close_date": "2018-01-21 21:55:00+00:00", "trade_duration": 40, "open_rate": 3.903e-05, "close_rate": 3.922563909774435e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2562.1316935690497, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 00:35:00+00:00", "close_date": "2018-01-22 10:35:00+00:00", "trade_duration": 600, "open_rate": 5.236e-05, "close_rate": 5.262245614035087e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1909.8548510313217, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": 0.0, "open_date": "2018-01-22 01:30:00+00:00", "close_date": "2018-01-22 02:10:00+00:00", "trade_duration": 40, "open_rate": 9.028e-05, "close_rate": 9.07325313283208e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1107.6650420912717, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 12:25:00+00:00", "close_date": "2018-01-22 14:35:00+00:00", "trade_duration": 130, "open_rate": 0.002687, "close_rate": 0.002700468671679198, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 37.21622627465575, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 13:15:00+00:00", "close_date": "2018-01-22 13:55:00+00:00", "trade_duration": 40, "open_rate": 4.168e-05, "close_rate": 4.188892230576441e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2399.232245681382, "profit_abs": 1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-22 14:00:00+00:00", "close_date": "2018-01-22 14:30:00+00:00", "trade_duration": 30, "open_rate": 8.821e-05, "close_rate": 8.953646616541353e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1133.6583153837435, "profit_abs": 0.0010000000000000148}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 15:55:00+00:00", "close_date": "2018-01-22 16:40:00+00:00", "trade_duration": 45, "open_rate": 5.172e-05, "close_rate": 5.1979248120300745e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1933.4880123743235, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-22 16:05:00+00:00", "close_date": "2018-01-22 16:25:00+00:00", "trade_duration": 20, "open_rate": 3.026e-05, "close_rate": 3.101839598997494e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3304.692663582287, "profit_abs": 0.0020000000000000157}, {"pair": "DASH/BTC", "profit_percent": 0.0, "open_date": "2018-01-22 19:50:00+00:00", "close_date": "2018-01-23 00:10:00+00:00", "trade_duration": 260, "open_rate": 0.07064, "close_rate": 0.07099408521303258, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.415628539071348, "profit_abs": 1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-22 21:25:00+00:00", "close_date": "2018-01-22 22:05:00+00:00", "trade_duration": 40, "open_rate": 0.01644483, "close_rate": 0.01652726022556391, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.080938507725528, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-23 00:05:00+00:00", "close_date": "2018-01-23 00:35:00+00:00", "trade_duration": 30, "open_rate": 4.331e-05, "close_rate": 4.3961278195488714e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2308.935580697299, "profit_abs": 0.0010000000000000148}, {"pair": "NXT/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-23 01:50:00+00:00", "close_date": "2018-01-23 02:15:00+00:00", "trade_duration": 25, "open_rate": 3.2e-05, "close_rate": 3.2802005012531326e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3125.0000000000005, "profit_abs": 0.0020000000000000018}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 04:25:00+00:00", "close_date": "2018-01-23 05:15:00+00:00", "trade_duration": 50, "open_rate": 0.09167706, "close_rate": 0.09213659413533835, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0907854156754153, "profit_abs": 1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 07:35:00+00:00", "close_date": "2018-01-23 09:00:00+00:00", "trade_duration": 85, "open_rate": 0.0692498, "close_rate": 0.06959691679197995, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4440474918339115, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 10:50:00+00:00", "close_date": "2018-01-23 13:05:00+00:00", "trade_duration": 135, "open_rate": 3.182e-05, "close_rate": 3.197949874686716e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3142.677561282213, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 11:05:00+00:00", "close_date": "2018-01-23 16:05:00+00:00", "trade_duration": 300, "open_rate": 0.04088, "close_rate": 0.04108491228070175, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4461839530332683, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 14:55:00+00:00", "close_date": "2018-01-23 15:35:00+00:00", "trade_duration": 40, "open_rate": 5.15e-05, "close_rate": 5.175814536340851e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1941.747572815534, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 16:35:00+00:00", "close_date": "2018-01-24 00:05:00+00:00", "trade_duration": 450, "open_rate": 0.09071698, "close_rate": 0.09117170170426064, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1023294646713329, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 17:25:00+00:00", "close_date": "2018-01-23 18:45:00+00:00", "trade_duration": 80, "open_rate": 3.128e-05, "close_rate": 3.1436791979949865e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3196.9309462915603, "profit_abs": -2.7755575615628914e-17}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 20:15:00+00:00", "close_date": "2018-01-23 22:00:00+00:00", "trade_duration": 105, "open_rate": 9.555e-05, "close_rate": 9.602894736842104e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1046.5724751439038, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 22:30:00+00:00", "close_date": "2018-01-23 23:10:00+00:00", "trade_duration": 40, "open_rate": 0.04080001, "close_rate": 0.0410045213283208, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.450979791426522, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 23:50:00+00:00", "close_date": "2018-01-24 03:35:00+00:00", "trade_duration": 225, "open_rate": 5.163e-05, "close_rate": 5.18887969924812e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1936.8584156498162, "profit_abs": 1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-24 00:20:00+00:00", "close_date": "2018-01-24 01:50:00+00:00", "trade_duration": 90, "open_rate": 0.04040781, "close_rate": 0.04061035541353383, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.474769110228938, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-24 06:45:00+00:00", "close_date": "2018-01-24 07:25:00+00:00", "trade_duration": 40, "open_rate": 5.132e-05, "close_rate": 5.157724310776942e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1948.5580670303975, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-24 14:15:00+00:00", "close_date": "2018-01-24 14:25:00+00:00", "trade_duration": 10, "open_rate": 5.198e-05, "close_rate": 5.432496240601503e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1923.8168526356292, "profit_abs": 0.0040000000000000036}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-24 14:50:00+00:00", "close_date": "2018-01-24 16:35:00+00:00", "trade_duration": 105, "open_rate": 3.054e-05, "close_rate": 3.069308270676692e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3274.3942370661425, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": 0.0, "open_date": "2018-01-24 15:10:00+00:00", "close_date": "2018-01-24 16:15:00+00:00", "trade_duration": 65, "open_rate": 9.263e-05, "close_rate": 9.309431077694236e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1079.5638562020945, "profit_abs": 2.7755575615628914e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-24 22:40:00+00:00", "close_date": "2018-01-24 23:25:00+00:00", "trade_duration": 45, "open_rate": 5.514e-05, "close_rate": 5.54163909774436e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1813.5654697134569, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-25 00:50:00+00:00", "close_date": "2018-01-25 01:30:00+00:00", "trade_duration": 40, "open_rate": 4.921e-05, "close_rate": 4.9456666666666664e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2032.1072952651903, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.0, "open_date": "2018-01-25 08:15:00+00:00", "close_date": "2018-01-25 12:15:00+00:00", "trade_duration": 240, "open_rate": 0.0026, "close_rate": 0.002613032581453634, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.46153846153847, "profit_abs": 1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 10:25:00+00:00", "close_date": "2018-01-25 16:15:00+00:00", "trade_duration": 350, "open_rate": 0.02799871, "close_rate": 0.028139054411027563, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.571593119825878, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 11:00:00+00:00", "close_date": "2018-01-25 11:45:00+00:00", "trade_duration": 45, "open_rate": 0.04078902, "close_rate": 0.0409934762406015, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4516401717913303, "profit_abs": -1.3877787807814457e-17}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 13:05:00+00:00", "close_date": "2018-01-25 13:45:00+00:00", "trade_duration": 40, "open_rate": 2.89e-05, "close_rate": 2.904486215538847e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3460.2076124567475, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 13:20:00+00:00", "close_date": "2018-01-25 14:05:00+00:00", "trade_duration": 45, "open_rate": 0.041103, "close_rate": 0.04130903007518797, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4329124394813033, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-25 15:45:00+00:00", "close_date": "2018-01-25 16:15:00+00:00", "trade_duration": 30, "open_rate": 5.428e-05, "close_rate": 5.509624060150376e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1842.2991893883568, "profit_abs": 0.0010000000000000148}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 17:45:00+00:00", "close_date": "2018-01-25 23:15:00+00:00", "trade_duration": 330, "open_rate": 5.414e-05, "close_rate": 5.441137844611528e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1847.063169560399, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 21:15:00+00:00", "close_date": "2018-01-25 21:55:00+00:00", "trade_duration": 40, "open_rate": 0.04140777, "close_rate": 0.0416153277443609, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.415005686130888, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.0, "open_date": "2018-01-26 02:05:00+00:00", "close_date": "2018-01-26 02:45:00+00:00", "trade_duration": 40, "open_rate": 0.00254309, "close_rate": 0.002555837318295739, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 39.32224183965177, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-26 02:55:00+00:00", "close_date": "2018-01-26 15:10:00+00:00", "trade_duration": 735, "open_rate": 5.607e-05, "close_rate": 5.6351052631578935e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1783.4849295523454, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.0, "open_date": "2018-01-26 06:10:00+00:00", "close_date": "2018-01-26 09:25:00+00:00", "trade_duration": 195, "open_rate": 0.00253806, "close_rate": 0.0025507821052631577, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 39.400171784748984, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-26 07:25:00+00:00", "close_date": "2018-01-26 09:55:00+00:00", "trade_duration": 150, "open_rate": 0.0415, "close_rate": 0.04170802005012531, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4096385542168677, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-26 09:55:00+00:00", "close_date": "2018-01-26 10:25:00+00:00", "trade_duration": 30, "open_rate": 5.321e-05, "close_rate": 5.401015037593984e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1879.3459875963165, "profit_abs": 0.000999999999999987}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-26 16:05:00+00:00", "close_date": "2018-01-26 16:45:00+00:00", "trade_duration": 40, "open_rate": 0.02772046, "close_rate": 0.02785940967418546, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.6074437437185387, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-26 23:35:00+00:00", "close_date": "2018-01-27 00:15:00+00:00", "trade_duration": 40, "open_rate": 0.09461341, "close_rate": 0.09508766268170424, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0569326272036914, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 00:35:00+00:00", "close_date": "2018-01-27 01:30:00+00:00", "trade_duration": 55, "open_rate": 5.615e-05, "close_rate": 5.643145363408521e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1780.9439002671415, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.07877175, "open_date": "2018-01-27 00:45:00+00:00", "close_date": "2018-01-30 04:45:00+00:00", "trade_duration": 4560, "open_rate": 5.556e-05, "close_rate": 5.144e-05, "open_at_end": true, "sell_reason": "force_sell", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1799.8560115190785, "profit_abs": -0.007896868250539965}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 02:30:00+00:00", "close_date": "2018-01-27 11:25:00+00:00", "trade_duration": 535, "open_rate": 0.06900001, "close_rate": 0.06934587471177944, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4492751522789635, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 06:25:00+00:00", "close_date": "2018-01-27 07:05:00+00:00", "trade_duration": 40, "open_rate": 0.09449985, "close_rate": 0.0949735334586466, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.058202737887944, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.04815133, "open_date": "2018-01-27 09:40:00+00:00", "close_date": "2018-01-30 04:40:00+00:00", "trade_duration": 4020, "open_rate": 0.0410697, "close_rate": 0.03928809, "open_at_end": true, "sell_reason": "force_sell", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4348850855983852, "profit_abs": -0.004827170578309559}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 11:45:00+00:00", "close_date": "2018-01-27 12:30:00+00:00", "trade_duration": 45, "open_rate": 0.0285, "close_rate": 0.02864285714285714, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.5087719298245617, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 12:35:00+00:00", "close_date": "2018-01-27 15:25:00+00:00", "trade_duration": 170, "open_rate": 0.02866372, "close_rate": 0.02880739779448621, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.4887307020861216, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 15:50:00+00:00", "close_date": "2018-01-27 16:50:00+00:00", "trade_duration": 60, "open_rate": 0.095381, "close_rate": 0.09585910025062656, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0484268355332824, "profit_abs": 1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 17:05:00+00:00", "close_date": "2018-01-27 17:45:00+00:00", "trade_duration": 40, "open_rate": 0.06759092, "close_rate": 0.06792972160401002, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4794886650455417, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 23:40:00+00:00", "close_date": "2018-01-28 01:05:00+00:00", "trade_duration": 85, "open_rate": 0.00258501, "close_rate": 0.002597967443609022, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.684569885609726, "profit_abs": -1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-28 02:25:00+00:00", "close_date": "2018-01-28 08:10:00+00:00", "trade_duration": 345, "open_rate": 0.06698502, "close_rate": 0.0673207845112782, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4928710926711672, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-28 10:25:00+00:00", "close_date": "2018-01-28 16:30:00+00:00", "trade_duration": 365, "open_rate": 0.0677177, "close_rate": 0.06805713709273183, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4767187899175547, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-28 20:35:00+00:00", "close_date": "2018-01-28 21:35:00+00:00", "trade_duration": 60, "open_rate": 5.215e-05, "close_rate": 5.2411403508771925e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1917.5455417066157, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-28 22:00:00+00:00", "close_date": "2018-01-28 22:30:00+00:00", "trade_duration": 30, "open_rate": 0.00273809, "close_rate": 0.002779264285714285, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 36.5218089982433, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-29 00:00:00+00:00", "close_date": "2018-01-29 00:30:00+00:00", "trade_duration": 30, "open_rate": 0.00274632, "close_rate": 0.002787618045112782, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 36.412362725392526, "profit_abs": 0.0010000000000000148}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-29 02:15:00+00:00", "close_date": "2018-01-29 03:00:00+00:00", "trade_duration": 45, "open_rate": 0.01622478, "close_rate": 0.016306107218045113, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.163411768911504, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 03:05:00+00:00", "close_date": "2018-01-29 03:45:00+00:00", "trade_duration": 40, "open_rate": 0.069, "close_rate": 0.06934586466165413, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4492753623188406, "profit_abs": -1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 05:20:00+00:00", "close_date": "2018-01-29 06:55:00+00:00", "trade_duration": 95, "open_rate": 8.755e-05, "close_rate": 8.798884711779448e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1142.204454597373, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 07:00:00+00:00", "close_date": "2018-01-29 19:25:00+00:00", "trade_duration": 745, "open_rate": 0.06825763, "close_rate": 0.06859977350877192, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4650376815016872, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 19:45:00+00:00", "close_date": "2018-01-29 20:25:00+00:00", "trade_duration": 40, "open_rate": 0.06713892, "close_rate": 0.06747545593984962, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4894490408841845, "profit_abs": -1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": -0.0199116, "open_date": "2018-01-29 23:30:00+00:00", "close_date": "2018-01-30 04:45:00+00:00", "trade_duration": 315, "open_rate": 8.934e-05, "close_rate": 8.8e-05, "open_at_end": true, "sell_reason": "force_sell", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1119.3194537721067, "profit_abs": -0.0019961383478844796}], "results_per_pair": [{"key": "TRX/BTC", "trades": 15, "profit_mean": 0.0023467073333333323, "profit_mean_pct": 0.23467073333333321, "profit_sum": 0.035200609999999986, "profit_sum_pct": 3.5200609999999988, "profit_total_abs": 0.0035288616521155086, "profit_total_pct": 1.1733536666666662, "duration_avg": "2:28:00", "wins": 9, "draws": 2, "losses": 4}, {"key": "ADA/BTC", "trades": 29, "profit_mean": -0.0011598141379310352, "profit_mean_pct": -0.11598141379310352, "profit_sum": -0.03363461000000002, "profit_sum_pct": -3.3634610000000023, "profit_total_abs": -0.0033718682505400333, "profit_total_pct": -1.1211536666666675, "duration_avg": "5:35:00", "wins": 9, "draws": 11, "losses": 9}, {"key": "XLM/BTC", "trades": 21, "profit_mean": 0.0026243899999999994, "profit_mean_pct": 0.2624389999999999, "profit_sum": 0.05511218999999999, "profit_sum_pct": 5.511218999999999, "profit_total_abs": 0.005525000000000002, "profit_total_pct": 1.8370729999999995, "duration_avg": "3:21:00", "wins": 12, "draws": 3, "losses": 6}, {"key": "ETH/BTC", "trades": 21, "profit_mean": 0.0009500057142857142, "profit_mean_pct": 0.09500057142857142, "profit_sum": 0.01995012, "profit_sum_pct": 1.9950119999999998, "profit_total_abs": 0.0019999999999999463, "profit_total_pct": 0.6650039999999999, "duration_avg": "2:17:00", "wins": 5, "draws": 10, "losses": 6}, {"key": "XMR/BTC", "trades": 16, "profit_mean": -0.0027899012500000007, "profit_mean_pct": -0.2789901250000001, "profit_sum": -0.04463842000000001, "profit_sum_pct": -4.463842000000001, "profit_total_abs": -0.0044750000000000345, "profit_total_pct": -1.4879473333333337, "duration_avg": "8:41:00", "wins": 6, "draws": 5, "losses": 5}, {"key": "ZEC/BTC", "trades": 21, "profit_mean": -0.00039290904761904774, "profit_mean_pct": -0.03929090476190478, "profit_sum": -0.008251090000000003, "profit_sum_pct": -0.8251090000000003, "profit_total_abs": -0.000827170578309569, "profit_total_pct": -0.27503633333333344, "duration_avg": "4:17:00", "wins": 8, "draws": 7, "losses": 6}, {"key": "NXT/BTC", "trades": 12, "profit_mean": -0.0012261025000000006, "profit_mean_pct": -0.12261025000000006, "profit_sum": -0.014713230000000008, "profit_sum_pct": -1.4713230000000008, "profit_total_abs": -0.0014750000000000874, "profit_total_pct": -0.4904410000000003, "duration_avg": "0:57:00", "wins": 4, "draws": 3, "losses": 5}, {"key": "LTC/BTC", "trades": 8, "profit_mean": 0.00748129625, "profit_mean_pct": 0.748129625, "profit_sum": 0.05985037, "profit_sum_pct": 5.985037, "profit_total_abs": 0.006000000000000019, "profit_total_pct": 1.9950123333333334, "duration_avg": "1:59:00", "wins": 5, "draws": 2, "losses": 1}, {"key": "ETC/BTC", "trades": 20, "profit_mean": 0.0022568569999999997, "profit_mean_pct": 0.22568569999999996, "profit_sum": 0.04513713999999999, "profit_sum_pct": 4.513713999999999, "profit_total_abs": 0.004525000000000001, "profit_total_pct": 1.504571333333333, "duration_avg": "1:45:00", "wins": 11, "draws": 4, "losses": 5}, {"key": "DASH/BTC", "trades": 16, "profit_mean": 0.0018703237499999997, "profit_mean_pct": 0.18703237499999997, "profit_sum": 0.029925179999999996, "profit_sum_pct": 2.9925179999999996, "profit_total_abs": 0.002999999999999961, "profit_total_pct": 0.9975059999999999, "duration_avg": "3:03:00", "wins": 4, "draws": 7, "losses": 5}, {"key": "TOTAL", "trades": 179, "profit_mean": 0.0008041243575418989, "profit_mean_pct": 0.0804124357541899, "profit_sum": 0.1439382599999999, "profit_sum_pct": 14.39382599999999, "profit_total_abs": 0.014429822823265714, "profit_total_pct": 4.797941999999996, "duration_avg": "3:40:00", "wins": 73, "draws": 54, "losses": 52}], "sell_reason_summary": [{"sell_reason": "roi", "trades": 170, "wins": 73, "draws": 54, "losses": 43, "profit_mean": 0.005398268352941177, "profit_mean_pct": 0.54, "profit_sum": 0.91770562, "profit_sum_pct": 91.77, "profit_total_abs": 0.09199999999999964, "profit_pct_total": 30.59}, {"sell_reason": "stop_loss", "trades": 6, "wins": 0, "draws": 0, "losses": 6, "profit_mean": -0.10448878000000002, "profit_mean_pct": -10.45, "profit_sum": -0.6269326800000001, "profit_sum_pct": -62.69, "profit_total_abs": -0.06284999999999992, "profit_pct_total": -20.9}, {"sell_reason": "force_sell", "trades": 3, "wins": 0, "draws": 0, "losses": 3, "profit_mean": -0.04894489333333333, "profit_mean_pct": -4.89, "profit_sum": -0.14683468, "profit_sum_pct": -14.68, "profit_total_abs": -0.014720177176734003, "profit_pct_total": -4.89}], "left_open_trades": [{"key": "TRX/BTC", "trades": 1, "profit_mean": -0.0199116, "profit_mean_pct": -1.9911600000000003, "profit_sum": -0.0199116, "profit_sum_pct": -1.9911600000000003, "profit_total_abs": -0.0019961383478844796, "profit_total_pct": -0.6637200000000001, "duration_avg": "5:15:00", "wins": 0, "draws": 0, "losses": 1}, {"key": "ADA/BTC", "trades": 1, "profit_mean": -0.07877175, "profit_mean_pct": -7.877175, "profit_sum": -0.07877175, "profit_sum_pct": -7.877175, "profit_total_abs": -0.007896868250539965, "profit_total_pct": -2.625725, "duration_avg": "3 days, 4:00:00", "wins": 0, "draws": 0, "losses": 1}, {"key": "ZEC/BTC", "trades": 1, "profit_mean": -0.04815133, "profit_mean_pct": -4.815133, "profit_sum": -0.04815133, "profit_sum_pct": -4.815133, "profit_total_abs": -0.004827170578309559, "profit_total_pct": -1.6050443333333335, "duration_avg": "2 days, 19:00:00", "wins": 0, "draws": 0, "losses": 1}, {"key": "TOTAL", "trades": 3, "profit_mean": -0.04894489333333333, "profit_mean_pct": -4.894489333333333, "profit_sum": -0.14683468, "profit_sum_pct": -14.683468, "profit_total_abs": -0.014720177176734003, "profit_total_pct": -4.8944893333333335, "duration_avg": "2 days, 1:25:00", "wins": 0, "draws": 0, "losses": 3}], "total_trades": 179, "backtest_start": "2018-01-30 04:45:00+00:00", "backtest_start_ts": 1517287500, "backtest_end": "2018-01-30 04:45:00+00:00", "backtest_end_ts": 1517287500, "backtest_days": 0, "trades_per_day": null, "market_change": 0.25, "stake_amount": 0.1, "max_drawdown": 0.21142322000000008, "drawdown_start": "2018-01-24 14:25:00+00:00", "drawdown_start_ts": 1516803900.0, "drawdown_end": "2018-01-30 04:45:00+00:00", "drawdown_end_ts": 1517287500.0,"pairlist": ["TRX/BTC", "ADA/BTC", "XLM/BTC", "ETH/BTC", "XMR/BTC", "ZEC/BTC","NXT/BTC", "LTC/BTC", "ETC/BTC", "DASH/BTC"]}}, "strategy_comparison": [{"key": "DefaultStrategy", "trades": 179, "profit_mean": 0.0008041243575418989, "profit_mean_pct": 0.0804124357541899, "profit_sum": 0.1439382599999999, "profit_sum_pct": 14.39382599999999, "profit_total_abs": 0.014429822823265714, "profit_total_pct": 4.797941999999996, "duration_avg": "3:40:00", "wins": 73, "draws": 54, "losses": 52}, {"key": "TestStrategy", "trades": 179, "profit_mean": 0.0008041243575418989, "profit_mean_pct": 0.0804124357541899, "profit_sum": 0.1439382599999999, "profit_sum_pct": 14.39382599999999, "profit_total_abs": 0.014429822823265714, "profit_total_pct": 4.797941999999996, "duration_avg": "3:40:00", "wins": 73, "draws": 54, "losses": 52}]}
diff --git a/tests/testdata/backtest-result_new.json b/tests/testdata/backtest-result_new.json
new file mode 100644
index 000000000..f004e879a
--- /dev/null
+++ b/tests/testdata/backtest-result_new.json
@@ -0,0 +1 @@
+{"strategy": {"DefaultStrategy": {"trades": [{"pair": "TRX/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:15:00+00:00", "close_date": "2018-01-10 07:20:00+00:00", "trade_duration": 5, "open_rate": 9.64e-05, "close_rate": 0.00010074887218045112, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1037.344398340249, "profit_abs": 0.00399999999999999}, {"pair": "ADA/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:15:00+00:00", "close_date": "2018-01-10 07:30:00+00:00", "trade_duration": 15, "open_rate": 4.756e-05, "close_rate": 4.9705563909774425e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2102.6072329688814, "profit_abs": 0.00399999999999999}, {"pair": "XLM/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:25:00+00:00", "close_date": "2018-01-10 07:35:00+00:00", "trade_duration": 10, "open_rate": 3.339e-05, "close_rate": 3.489631578947368e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2994.908655286014, "profit_abs": 0.0040000000000000036}, {"pair": "TRX/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-10 07:25:00+00:00", "close_date": "2018-01-10 07:40:00+00:00", "trade_duration": 15, "open_rate": 9.696e-05, "close_rate": 0.00010133413533834584, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1031.3531353135315, "profit_abs": 0.00399999999999999}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 07:35:00+00:00", "close_date": "2018-01-10 08:35:00+00:00", "trade_duration": 60, "open_rate": 0.0943, "close_rate": 0.09477268170426063, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0604453870625663, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-10 07:40:00+00:00", "close_date": "2018-01-10 08:10:00+00:00", "trade_duration": 30, "open_rate": 0.02719607, "close_rate": 0.02760503345864661, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.677001860930642, "profit_abs": 0.0010000000000000009}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-10 08:15:00+00:00", "close_date": "2018-01-10 09:55:00+00:00", "trade_duration": 100, "open_rate": 0.04634952, "close_rate": 0.046581848421052625, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.1575196463739, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 14:45:00+00:00", "close_date": "2018-01-10 15:50:00+00:00", "trade_duration": 65, "open_rate": 3.066e-05, "close_rate": 3.081368421052631e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3261.5786040443577, "profit_abs": -1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-10 16:35:00+00:00", "close_date": "2018-01-10 17:15:00+00:00", "trade_duration": 40, "open_rate": 0.0168999, "close_rate": 0.016984611278195488, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 5.917194776300452, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 16:40:00+00:00", "close_date": "2018-01-10 17:20:00+00:00", "trade_duration": 40, "open_rate": 0.09132568, "close_rate": 0.0917834528320802, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0949822656672252, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 18:50:00+00:00", "close_date": "2018-01-10 19:45:00+00:00", "trade_duration": 55, "open_rate": 0.08898003, "close_rate": 0.08942604518796991, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1238476768326557, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-10 22:15:00+00:00", "close_date": "2018-01-10 23:00:00+00:00", "trade_duration": 45, "open_rate": 0.08560008, "close_rate": 0.08602915308270676, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1682232072680307, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-10 22:50:00+00:00", "close_date": "2018-01-10 23:20:00+00:00", "trade_duration": 30, "open_rate": 0.00249083, "close_rate": 0.0025282860902255634, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 40.147260150231055, "profit_abs": 0.000999999999999987}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-10 23:15:00+00:00", "close_date": "2018-01-11 00:15:00+00:00", "trade_duration": 60, "open_rate": 3.022e-05, "close_rate": 3.037147869674185e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3309.0668431502318, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-10 23:40:00+00:00", "close_date": "2018-01-11 00:05:00+00:00", "trade_duration": 25, "open_rate": 0.002437, "close_rate": 0.0024980776942355883, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 41.03405826836274, "profit_abs": 0.001999999999999974}, {"pair": "ZEC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 00:00:00+00:00", "close_date": "2018-01-11 00:35:00+00:00", "trade_duration": 35, "open_rate": 0.04771803, "close_rate": 0.04843559436090225, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.0956439316543456, "profit_abs": 0.0010000000000000009}, {"pair": "XLM/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-11 03:40:00+00:00", "close_date": "2018-01-11 04:25:00+00:00", "trade_duration": 45, "open_rate": 3.651e-05, "close_rate": 3.2859000000000005e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2738.9756231169545, "profit_abs": -0.01047499999999997}, {"pair": "ETH/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 03:55:00+00:00", "close_date": "2018-01-11 04:25:00+00:00", "trade_duration": 30, "open_rate": 0.08824105, "close_rate": 0.08956798308270676, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1332594070446804, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 04:00:00+00:00", "close_date": "2018-01-11 04:50:00+00:00", "trade_duration": 50, "open_rate": 0.00243, "close_rate": 0.002442180451127819, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 41.1522633744856, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 04:30:00+00:00", "close_date": "2018-01-11 04:55:00+00:00", "trade_duration": 25, "open_rate": 0.04545064, "close_rate": 0.046589753784461146, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.200189040242338, "profit_abs": 0.001999999999999988}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 04:30:00+00:00", "close_date": "2018-01-11 04:50:00+00:00", "trade_duration": 20, "open_rate": 3.372e-05, "close_rate": 3.456511278195488e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2965.599051008304, "profit_abs": 0.001999999999999988}, {"pair": "XMR/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 04:55:00+00:00", "close_date": "2018-01-11 05:15:00+00:00", "trade_duration": 20, "open_rate": 0.02644, "close_rate": 0.02710265664160401, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.7821482602118004, "profit_abs": 0.001999999999999988}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 11:20:00+00:00", "close_date": "2018-01-11 12:00:00+00:00", "trade_duration": 40, "open_rate": 0.08812, "close_rate": 0.08856170426065162, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1348161597821154, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 11:35:00+00:00", "close_date": "2018-01-11 12:15:00+00:00", "trade_duration": 40, "open_rate": 0.02683577, "close_rate": 0.026970285137844607, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.7263696923919087, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-11 14:00:00+00:00", "close_date": "2018-01-11 14:25:00+00:00", "trade_duration": 25, "open_rate": 4.919e-05, "close_rate": 5.04228320802005e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2032.9335230737956, "profit_abs": 0.0020000000000000018}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 19:25:00+00:00", "close_date": "2018-01-11 20:35:00+00:00", "trade_duration": 70, "open_rate": 0.08784896, "close_rate": 0.08828930566416039, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1383174029607181, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 22:35:00+00:00", "close_date": "2018-01-11 23:30:00+00:00", "trade_duration": 55, "open_rate": 5.105e-05, "close_rate": 5.130588972431077e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1958.8638589618022, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 22:55:00+00:00", "close_date": "2018-01-11 23:25:00+00:00", "trade_duration": 30, "open_rate": 3.96e-05, "close_rate": 4.019548872180451e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2525.252525252525, "profit_abs": 0.0010000000000000148}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 22:55:00+00:00", "close_date": "2018-01-11 23:35:00+00:00", "trade_duration": 40, "open_rate": 2.885e-05, "close_rate": 2.899461152882205e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3466.204506065858, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-11 23:30:00+00:00", "close_date": "2018-01-12 00:05:00+00:00", "trade_duration": 35, "open_rate": 0.02645, "close_rate": 0.026847744360902256, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.780718336483932, "profit_abs": 0.0010000000000000148}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-11 23:55:00+00:00", "close_date": "2018-01-12 01:15:00+00:00", "trade_duration": 80, "open_rate": 0.048, "close_rate": 0.04824060150375939, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.0833333333333335, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-12 21:15:00+00:00", "close_date": "2018-01-12 21:40:00+00:00", "trade_duration": 25, "open_rate": 4.692e-05, "close_rate": 4.809593984962405e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2131.287297527707, "profit_abs": 0.001999999999999974}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 00:55:00+00:00", "close_date": "2018-01-13 06:20:00+00:00", "trade_duration": 325, "open_rate": 0.00256966, "close_rate": 0.0025825405012531327, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.91565421106294, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.0, "open_date": "2018-01-13 10:55:00+00:00", "close_date": "2018-01-13 11:35:00+00:00", "trade_duration": 40, "open_rate": 6.262e-05, "close_rate": 6.293388471177944e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1596.933886937081, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-13 13:05:00+00:00", "close_date": "2018-01-15 14:10:00+00:00", "trade_duration": 2945, "open_rate": 4.73e-05, "close_rate": 4.753709273182957e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2114.1649048625795, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 13:30:00+00:00", "close_date": "2018-01-13 14:45:00+00:00", "trade_duration": 75, "open_rate": 6.063e-05, "close_rate": 6.0933909774436085e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1649.348507339601, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 13:40:00+00:00", "close_date": "2018-01-13 23:30:00+00:00", "trade_duration": 590, "open_rate": 0.00011082, "close_rate": 0.00011137548872180448, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 902.3641941887746, "profit_abs": -2.7755575615628914e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 15:15:00+00:00", "close_date": "2018-01-13 15:55:00+00:00", "trade_duration": 40, "open_rate": 5.93e-05, "close_rate": 5.9597243107769415e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1686.3406408094436, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 16:30:00+00:00", "close_date": "2018-01-13 17:10:00+00:00", "trade_duration": 40, "open_rate": 0.04850003, "close_rate": 0.04874313791979949, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.0618543947292407, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-13 22:05:00+00:00", "close_date": "2018-01-14 06:25:00+00:00", "trade_duration": 500, "open_rate": 0.09825019, "close_rate": 0.09874267215538848, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0178097365511456, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": 0.0, "open_date": "2018-01-14 00:20:00+00:00", "close_date": "2018-01-14 22:55:00+00:00", "trade_duration": 1355, "open_rate": 6.018e-05, "close_rate": 6.048165413533834e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1661.681621801263, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-14 12:45:00+00:00", "close_date": "2018-01-14 13:25:00+00:00", "trade_duration": 40, "open_rate": 0.09758999, "close_rate": 0.0980791628822055, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.024695258191952, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-14 15:30:00+00:00", "close_date": "2018-01-14 16:00:00+00:00", "trade_duration": 30, "open_rate": 0.00311, "close_rate": 0.0031567669172932328, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 32.154340836012864, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-14 20:45:00+00:00", "close_date": "2018-01-14 22:15:00+00:00", "trade_duration": 90, "open_rate": 0.00312401, "close_rate": 0.003139669197994987, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 32.010140812609436, "profit_abs": -1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-14 23:35:00+00:00", "close_date": "2018-01-15 00:30:00+00:00", "trade_duration": 55, "open_rate": 0.0174679, "close_rate": 0.017555458395989976, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 5.724786608579165, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-14 23:45:00+00:00", "close_date": "2018-01-15 00:25:00+00:00", "trade_duration": 40, "open_rate": 0.07346846, "close_rate": 0.07383672295739348, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.3611282991367997, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 02:25:00+00:00", "close_date": "2018-01-15 03:05:00+00:00", "trade_duration": 40, "open_rate": 0.097994, "close_rate": 0.09848519799498744, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.020470641059657, "profit_abs": -2.7755575615628914e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 07:20:00+00:00", "close_date": "2018-01-15 08:00:00+00:00", "trade_duration": 40, "open_rate": 0.09659, "close_rate": 0.09707416040100247, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0353038616834043, "profit_abs": -2.7755575615628914e-17}, {"pair": "TRX/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-15 08:20:00+00:00", "close_date": "2018-01-15 08:55:00+00:00", "trade_duration": 35, "open_rate": 9.987e-05, "close_rate": 0.00010137180451127818, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1001.3016921998599, "profit_abs": 0.0010000000000000009}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-15 12:10:00+00:00", "close_date": "2018-01-16 02:50:00+00:00", "trade_duration": 880, "open_rate": 0.0948969, "close_rate": 0.09537257368421052, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0537752023511833, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 14:10:00+00:00", "close_date": "2018-01-15 17:40:00+00:00", "trade_duration": 210, "open_rate": 0.071, "close_rate": 0.07135588972431077, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4084507042253522, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 14:30:00+00:00", "close_date": "2018-01-15 15:10:00+00:00", "trade_duration": 40, "open_rate": 0.04600501, "close_rate": 0.046235611553884705, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.173676301776698, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 18:10:00+00:00", "close_date": "2018-01-15 19:25:00+00:00", "trade_duration": 75, "open_rate": 9.438e-05, "close_rate": 9.485308270676693e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1059.5465140919687, "profit_abs": 1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 18:35:00+00:00", "close_date": "2018-01-15 19:15:00+00:00", "trade_duration": 40, "open_rate": 0.03040001, "close_rate": 0.030552391002506264, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.2894726021471703, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-15 20:25:00+00:00", "close_date": "2018-01-16 08:25:00+00:00", "trade_duration": 720, "open_rate": 5.837e-05, "close_rate": 5.2533e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1713.2088401576154, "profit_abs": -0.010474999999999984}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-15 20:40:00+00:00", "close_date": "2018-01-15 22:00:00+00:00", "trade_duration": 80, "open_rate": 0.046036, "close_rate": 0.04626675689223057, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.1722130506560084, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-16 00:30:00+00:00", "close_date": "2018-01-16 01:10:00+00:00", "trade_duration": 40, "open_rate": 0.0028685, "close_rate": 0.0028828784461152877, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 34.86142583231654, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": 0.0, "open_date": "2018-01-16 01:15:00+00:00", "close_date": "2018-01-16 02:35:00+00:00", "trade_duration": 80, "open_rate": 0.06731755, "close_rate": 0.0676549813283208, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4854967241083492, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-16 07:45:00+00:00", "close_date": "2018-01-16 08:40:00+00:00", "trade_duration": 55, "open_rate": 0.09217614, "close_rate": 0.09263817578947368, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0848794492804754, "profit_abs": 0.0}, {"pair": "LTC/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 08:35:00+00:00", "close_date": "2018-01-16 08:55:00+00:00", "trade_duration": 20, "open_rate": 0.0165, "close_rate": 0.016913533834586467, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.0606060606060606, "profit_abs": 0.0020000000000000018}, {"pair": "TRX/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 08:35:00+00:00", "close_date": "2018-01-16 08:40:00+00:00", "trade_duration": 5, "open_rate": 7.953e-05, "close_rate": 8.311781954887218e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1257.387149503332, "profit_abs": 0.00399999999999999}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-16 08:45:00+00:00", "close_date": "2018-01-16 09:50:00+00:00", "trade_duration": 65, "open_rate": 0.045202, "close_rate": 0.04542857644110275, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.2122914915269236, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 09:15:00+00:00", "close_date": "2018-01-16 09:45:00+00:00", "trade_duration": 30, "open_rate": 5.248e-05, "close_rate": 5.326917293233082e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1905.487804878049, "profit_abs": 0.0010000000000000009}, {"pair": "XMR/BTC", "profit_percent": 0.0, "open_date": "2018-01-16 09:15:00+00:00", "close_date": "2018-01-16 09:55:00+00:00", "trade_duration": 40, "open_rate": 0.02892318, "close_rate": 0.02906815834586466, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.457434486802627, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 09:50:00+00:00", "close_date": "2018-01-16 10:10:00+00:00", "trade_duration": 20, "open_rate": 5.158e-05, "close_rate": 5.287273182957392e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1938.735944164405, "profit_abs": 0.001999999999999988}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 10:05:00+00:00", "close_date": "2018-01-16 10:35:00+00:00", "trade_duration": 30, "open_rate": 0.02828232, "close_rate": 0.02870761804511278, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.5357778286929786, "profit_abs": 0.0010000000000000009}, {"pair": "ZEC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 10:05:00+00:00", "close_date": "2018-01-16 10:40:00+00:00", "trade_duration": 35, "open_rate": 0.04357584, "close_rate": 0.044231115789473675, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.294849623093898, "profit_abs": 0.0010000000000000009}, {"pair": "ADA/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 13:45:00+00:00", "close_date": "2018-01-16 14:20:00+00:00", "trade_duration": 35, "open_rate": 5.362e-05, "close_rate": 5.442631578947368e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1864.975755315181, "profit_abs": 0.0010000000000000148}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-16 17:30:00+00:00", "close_date": "2018-01-16 18:25:00+00:00", "trade_duration": 55, "open_rate": 5.302e-05, "close_rate": 5.328576441102756e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1886.0807242549984, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 18:15:00+00:00", "close_date": "2018-01-16 18:45:00+00:00", "trade_duration": 30, "open_rate": 0.09129999, "close_rate": 0.09267292218045112, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0952903718828448, "profit_abs": 0.0010000000000000148}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 18:15:00+00:00", "close_date": "2018-01-16 18:35:00+00:00", "trade_duration": 20, "open_rate": 3.808e-05, "close_rate": 3.903438596491228e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2626.0504201680674, "profit_abs": 0.0020000000000000018}, {"pair": "XMR/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-16 19:00:00+00:00", "close_date": "2018-01-16 19:30:00+00:00", "trade_duration": 30, "open_rate": 0.02811012, "close_rate": 0.028532828571428567, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.557437677249333, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-16 21:25:00+00:00", "close_date": "2018-01-16 22:25:00+00:00", "trade_duration": 60, "open_rate": 0.00258379, "close_rate": 0.002325411, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.702835756775904, "profit_abs": -0.010474999999999984}, {"pair": "NXT/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-16 21:25:00+00:00", "close_date": "2018-01-16 22:45:00+00:00", "trade_duration": 80, "open_rate": 2.559e-05, "close_rate": 2.3031e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3907.7764751856193, "profit_abs": -0.010474999999999998}, {"pair": "TRX/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-16 21:35:00+00:00", "close_date": "2018-01-16 22:25:00+00:00", "trade_duration": 50, "open_rate": 7.62e-05, "close_rate": 6.858e-05, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1312.3359580052495, "profit_abs": -0.010474999999999984}, {"pair": "ETC/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:30:00+00:00", "close_date": "2018-01-16 22:35:00+00:00", "trade_duration": 5, "open_rate": 0.00229844, "close_rate": 0.002402129022556391, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 43.507770487809125, "profit_abs": 0.004000000000000017}, {"pair": "LTC/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:30:00+00:00", "close_date": "2018-01-16 22:40:00+00:00", "trade_duration": 10, "open_rate": 0.0151, "close_rate": 0.015781203007518795, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.622516556291391, "profit_abs": 0.00399999999999999}, {"pair": "ETC/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:40:00+00:00", "close_date": "2018-01-16 22:45:00+00:00", "trade_duration": 5, "open_rate": 0.00235676, "close_rate": 0.00246308, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 42.431134269081284, "profit_abs": 0.0040000000000000036}, {"pair": "DASH/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-16 22:45:00+00:00", "close_date": "2018-01-16 23:05:00+00:00", "trade_duration": 20, "open_rate": 0.0630692, "close_rate": 0.06464988170426066, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.585559988076589, "profit_abs": 0.0020000000000000018}, {"pair": "NXT/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-16 22:50:00+00:00", "close_date": "2018-01-16 22:55:00+00:00", "trade_duration": 5, "open_rate": 2.2e-05, "close_rate": 2.299248120300751e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 4545.454545454546, "profit_abs": 0.003999999999999976}, {"pair": "ADA/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-17 03:30:00+00:00", "close_date": "2018-01-17 04:00:00+00:00", "trade_duration": 30, "open_rate": 4.974e-05, "close_rate": 5.048796992481203e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2010.454362685967, "profit_abs": 0.0010000000000000009}, {"pair": "TRX/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-17 03:55:00+00:00", "close_date": "2018-01-17 04:15:00+00:00", "trade_duration": 20, "open_rate": 7.108e-05, "close_rate": 7.28614536340852e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1406.8655036578502, "profit_abs": 0.001999999999999974}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 09:35:00+00:00", "close_date": "2018-01-17 10:15:00+00:00", "trade_duration": 40, "open_rate": 0.04327, "close_rate": 0.04348689223057644, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.3110700254217704, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 10:20:00+00:00", "close_date": "2018-01-17 17:00:00+00:00", "trade_duration": 400, "open_rate": 4.997e-05, "close_rate": 5.022047619047618e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2001.2007204322595, "profit_abs": -1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 10:30:00+00:00", "close_date": "2018-01-17 11:25:00+00:00", "trade_duration": 55, "open_rate": 0.06836818, "close_rate": 0.06871087764411027, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4626687444363737, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 10:30:00+00:00", "close_date": "2018-01-17 11:10:00+00:00", "trade_duration": 40, "open_rate": 3.63e-05, "close_rate": 3.648195488721804e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2754.8209366391184, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 12:30:00+00:00", "close_date": "2018-01-17 22:05:00+00:00", "trade_duration": 575, "open_rate": 0.0281, "close_rate": 0.02824085213032581, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.5587188612099645, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-17 12:35:00+00:00", "close_date": "2018-01-17 16:55:00+00:00", "trade_duration": 260, "open_rate": 0.08651001, "close_rate": 0.08694364413533832, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1559355963546878, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 05:00:00+00:00", "close_date": "2018-01-18 05:55:00+00:00", "trade_duration": 55, "open_rate": 5.633e-05, "close_rate": 5.6612355889724306e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1775.2529735487308, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-18 05:20:00+00:00", "close_date": "2018-01-18 05:55:00+00:00", "trade_duration": 35, "open_rate": 0.06988494, "close_rate": 0.07093584135338346, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.430923457900944, "profit_abs": 0.0010000000000000009}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 07:35:00+00:00", "close_date": "2018-01-18 08:15:00+00:00", "trade_duration": 40, "open_rate": 5.545e-05, "close_rate": 5.572794486215538e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1803.4265103697026, "profit_abs": -1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 09:00:00+00:00", "close_date": "2018-01-18 09:40:00+00:00", "trade_duration": 40, "open_rate": 0.01633527, "close_rate": 0.016417151052631574, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.121723118136401, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 16:40:00+00:00", "close_date": "2018-01-18 17:20:00+00:00", "trade_duration": 40, "open_rate": 0.00269734, "close_rate": 0.002710860501253133, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 37.073561360451414, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-18 18:05:00+00:00", "close_date": "2018-01-18 18:30:00+00:00", "trade_duration": 25, "open_rate": 4.475e-05, "close_rate": 4.587155388471177e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2234.63687150838, "profit_abs": 0.0020000000000000018}, {"pair": "NXT/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-18 18:25:00+00:00", "close_date": "2018-01-18 18:55:00+00:00", "trade_duration": 30, "open_rate": 2.79e-05, "close_rate": 2.8319548872180444e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3584.2293906810037, "profit_abs": 0.000999999999999987}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 20:10:00+00:00", "close_date": "2018-01-18 20:50:00+00:00", "trade_duration": 40, "open_rate": 0.04439326, "close_rate": 0.04461578260651629, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.2525942001105577, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 21:30:00+00:00", "close_date": "2018-01-19 00:35:00+00:00", "trade_duration": 185, "open_rate": 4.49e-05, "close_rate": 4.51250626566416e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2227.1714922049, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-18 21:55:00+00:00", "close_date": "2018-01-19 05:05:00+00:00", "trade_duration": 430, "open_rate": 0.02855, "close_rate": 0.028693107769423555, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.502626970227671, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 22:10:00+00:00", "close_date": "2018-01-18 22:50:00+00:00", "trade_duration": 40, "open_rate": 5.796e-05, "close_rate": 5.8250526315789473e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1725.3278122843342, "profit_abs": 1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-18 23:50:00+00:00", "close_date": "2018-01-19 00:30:00+00:00", "trade_duration": 40, "open_rate": 0.04340323, "close_rate": 0.04362079005012531, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.303975994413319, "profit_abs": 1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-19 16:45:00+00:00", "close_date": "2018-01-19 17:35:00+00:00", "trade_duration": 50, "open_rate": 0.04454455, "close_rate": 0.04476783095238095, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.244943545282195, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-19 17:15:00+00:00", "close_date": "2018-01-19 19:55:00+00:00", "trade_duration": 160, "open_rate": 5.62e-05, "close_rate": 5.648170426065162e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1779.3594306049824, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-19 17:20:00+00:00", "close_date": "2018-01-19 20:15:00+00:00", "trade_duration": 175, "open_rate": 4.339e-05, "close_rate": 4.360749373433584e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2304.6784973496196, "profit_abs": -1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": 0.0, "open_date": "2018-01-20 04:45:00+00:00", "close_date": "2018-01-20 17:35:00+00:00", "trade_duration": 770, "open_rate": 0.0001009, "close_rate": 0.00010140576441102755, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 991.0802775024778, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 04:50:00+00:00", "close_date": "2018-01-20 15:15:00+00:00", "trade_duration": 625, "open_rate": 0.00270505, "close_rate": 0.002718609147869674, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 36.96789338459548, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 04:50:00+00:00", "close_date": "2018-01-20 07:00:00+00:00", "trade_duration": 130, "open_rate": 0.03000002, "close_rate": 0.030150396040100245, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.3333311111125927, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 09:00:00+00:00", "close_date": "2018-01-20 09:40:00+00:00", "trade_duration": 40, "open_rate": 5.46e-05, "close_rate": 5.4873684210526304e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1831.5018315018317, "profit_abs": -1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.10448878, "open_date": "2018-01-20 18:25:00+00:00", "close_date": "2018-01-25 03:50:00+00:00", "trade_duration": 6325, "open_rate": 0.03082222, "close_rate": 0.027739998, "open_at_end": false, "sell_reason": "stop_loss", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.244412634781012, "profit_abs": -0.010474999999999998}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-20 22:25:00+00:00", "close_date": "2018-01-20 23:15:00+00:00", "trade_duration": 50, "open_rate": 0.08969999, "close_rate": 0.09014961401002504, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1148273260677064, "profit_abs": 0.0}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-21 02:50:00+00:00", "close_date": "2018-01-21 14:30:00+00:00", "trade_duration": 700, "open_rate": 0.01632501, "close_rate": 0.01640683962406015, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.125570520324337, "profit_abs": 1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-21 10:20:00+00:00", "close_date": "2018-01-21 11:00:00+00:00", "trade_duration": 40, "open_rate": 0.070538, "close_rate": 0.07089157393483708, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.417675579120474, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-21 15:50:00+00:00", "close_date": "2018-01-21 18:45:00+00:00", "trade_duration": 175, "open_rate": 5.301e-05, "close_rate": 5.327571428571427e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1886.4365214110546, "profit_abs": -2.7755575615628914e-17}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-21 16:20:00+00:00", "close_date": "2018-01-21 17:00:00+00:00", "trade_duration": 40, "open_rate": 3.955e-05, "close_rate": 3.9748245614035085e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2528.4450063211125, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-21 21:15:00+00:00", "close_date": "2018-01-21 21:45:00+00:00", "trade_duration": 30, "open_rate": 0.00258505, "close_rate": 0.002623922932330827, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.6839712964933, "profit_abs": 0.0010000000000000009}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-21 21:15:00+00:00", "close_date": "2018-01-21 21:55:00+00:00", "trade_duration": 40, "open_rate": 3.903e-05, "close_rate": 3.922563909774435e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2562.1316935690497, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 00:35:00+00:00", "close_date": "2018-01-22 10:35:00+00:00", "trade_duration": 600, "open_rate": 5.236e-05, "close_rate": 5.262245614035087e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1909.8548510313217, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": 0.0, "open_date": "2018-01-22 01:30:00+00:00", "close_date": "2018-01-22 02:10:00+00:00", "trade_duration": 40, "open_rate": 9.028e-05, "close_rate": 9.07325313283208e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1107.6650420912717, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 12:25:00+00:00", "close_date": "2018-01-22 14:35:00+00:00", "trade_duration": 130, "open_rate": 0.002687, "close_rate": 0.002700468671679198, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 37.21622627465575, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 13:15:00+00:00", "close_date": "2018-01-22 13:55:00+00:00", "trade_duration": 40, "open_rate": 4.168e-05, "close_rate": 4.188892230576441e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2399.232245681382, "profit_abs": 1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-22 14:00:00+00:00", "close_date": "2018-01-22 14:30:00+00:00", "trade_duration": 30, "open_rate": 8.821e-05, "close_rate": 8.953646616541353e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1133.6583153837435, "profit_abs": 0.0010000000000000148}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-22 15:55:00+00:00", "close_date": "2018-01-22 16:40:00+00:00", "trade_duration": 45, "open_rate": 5.172e-05, "close_rate": 5.1979248120300745e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1933.4880123743235, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-22 16:05:00+00:00", "close_date": "2018-01-22 16:25:00+00:00", "trade_duration": 20, "open_rate": 3.026e-05, "close_rate": 3.101839598997494e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3304.692663582287, "profit_abs": 0.0020000000000000157}, {"pair": "DASH/BTC", "profit_percent": 0.0, "open_date": "2018-01-22 19:50:00+00:00", "close_date": "2018-01-23 00:10:00+00:00", "trade_duration": 260, "open_rate": 0.07064, "close_rate": 0.07099408521303258, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.415628539071348, "profit_abs": 1.3877787807814457e-17}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-22 21:25:00+00:00", "close_date": "2018-01-22 22:05:00+00:00", "trade_duration": 40, "open_rate": 0.01644483, "close_rate": 0.01652726022556391, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.080938507725528, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-23 00:05:00+00:00", "close_date": "2018-01-23 00:35:00+00:00", "trade_duration": 30, "open_rate": 4.331e-05, "close_rate": 4.3961278195488714e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2308.935580697299, "profit_abs": 0.0010000000000000148}, {"pair": "NXT/BTC", "profit_percent": 0.01995012, "open_date": "2018-01-23 01:50:00+00:00", "close_date": "2018-01-23 02:15:00+00:00", "trade_duration": 25, "open_rate": 3.2e-05, "close_rate": 3.2802005012531326e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3125.0000000000005, "profit_abs": 0.0020000000000000018}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 04:25:00+00:00", "close_date": "2018-01-23 05:15:00+00:00", "trade_duration": 50, "open_rate": 0.09167706, "close_rate": 0.09213659413533835, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0907854156754153, "profit_abs": 1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 07:35:00+00:00", "close_date": "2018-01-23 09:00:00+00:00", "trade_duration": 85, "open_rate": 0.0692498, "close_rate": 0.06959691679197995, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4440474918339115, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 10:50:00+00:00", "close_date": "2018-01-23 13:05:00+00:00", "trade_duration": 135, "open_rate": 3.182e-05, "close_rate": 3.197949874686716e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3142.677561282213, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 11:05:00+00:00", "close_date": "2018-01-23 16:05:00+00:00", "trade_duration": 300, "open_rate": 0.04088, "close_rate": 0.04108491228070175, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4461839530332683, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 14:55:00+00:00", "close_date": "2018-01-23 15:35:00+00:00", "trade_duration": 40, "open_rate": 5.15e-05, "close_rate": 5.175814536340851e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1941.747572815534, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-23 16:35:00+00:00", "close_date": "2018-01-24 00:05:00+00:00", "trade_duration": 450, "open_rate": 0.09071698, "close_rate": 0.09117170170426064, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.1023294646713329, "profit_abs": 0.0}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 17:25:00+00:00", "close_date": "2018-01-23 18:45:00+00:00", "trade_duration": 80, "open_rate": 3.128e-05, "close_rate": 3.1436791979949865e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3196.9309462915603, "profit_abs": -2.7755575615628914e-17}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 20:15:00+00:00", "close_date": "2018-01-23 22:00:00+00:00", "trade_duration": 105, "open_rate": 9.555e-05, "close_rate": 9.602894736842104e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1046.5724751439038, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 22:30:00+00:00", "close_date": "2018-01-23 23:10:00+00:00", "trade_duration": 40, "open_rate": 0.04080001, "close_rate": 0.0410045213283208, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.450979791426522, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-23 23:50:00+00:00", "close_date": "2018-01-24 03:35:00+00:00", "trade_duration": 225, "open_rate": 5.163e-05, "close_rate": 5.18887969924812e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1936.8584156498162, "profit_abs": 1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": 0.0, "open_date": "2018-01-24 00:20:00+00:00", "close_date": "2018-01-24 01:50:00+00:00", "trade_duration": 90, "open_rate": 0.04040781, "close_rate": 0.04061035541353383, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.474769110228938, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-24 06:45:00+00:00", "close_date": "2018-01-24 07:25:00+00:00", "trade_duration": 40, "open_rate": 5.132e-05, "close_rate": 5.157724310776942e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1948.5580670303975, "profit_abs": 0.0}, {"pair": "ADA/BTC", "profit_percent": 0.03990025, "open_date": "2018-01-24 14:15:00+00:00", "close_date": "2018-01-24 14:25:00+00:00", "trade_duration": 10, "open_rate": 5.198e-05, "close_rate": 5.432496240601503e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1923.8168526356292, "profit_abs": 0.0040000000000000036}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-24 14:50:00+00:00", "close_date": "2018-01-24 16:35:00+00:00", "trade_duration": 105, "open_rate": 3.054e-05, "close_rate": 3.069308270676692e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3274.3942370661425, "profit_abs": 0.0}, {"pair": "TRX/BTC", "profit_percent": 0.0, "open_date": "2018-01-24 15:10:00+00:00", "close_date": "2018-01-24 16:15:00+00:00", "trade_duration": 65, "open_rate": 9.263e-05, "close_rate": 9.309431077694236e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1079.5638562020945, "profit_abs": 2.7755575615628914e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-24 22:40:00+00:00", "close_date": "2018-01-24 23:25:00+00:00", "trade_duration": 45, "open_rate": 5.514e-05, "close_rate": 5.54163909774436e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1813.5654697134569, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-25 00:50:00+00:00", "close_date": "2018-01-25 01:30:00+00:00", "trade_duration": 40, "open_rate": 4.921e-05, "close_rate": 4.9456666666666664e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2032.1072952651903, "profit_abs": 1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.0, "open_date": "2018-01-25 08:15:00+00:00", "close_date": "2018-01-25 12:15:00+00:00", "trade_duration": 240, "open_rate": 0.0026, "close_rate": 0.002613032581453634, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.46153846153847, "profit_abs": 1.3877787807814457e-17}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 10:25:00+00:00", "close_date": "2018-01-25 16:15:00+00:00", "trade_duration": 350, "open_rate": 0.02799871, "close_rate": 0.028139054411027563, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.571593119825878, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 11:00:00+00:00", "close_date": "2018-01-25 11:45:00+00:00", "trade_duration": 45, "open_rate": 0.04078902, "close_rate": 0.0409934762406015, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4516401717913303, "profit_abs": -1.3877787807814457e-17}, {"pair": "NXT/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 13:05:00+00:00", "close_date": "2018-01-25 13:45:00+00:00", "trade_duration": 40, "open_rate": 2.89e-05, "close_rate": 2.904486215538847e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3460.2076124567475, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 13:20:00+00:00", "close_date": "2018-01-25 14:05:00+00:00", "trade_duration": 45, "open_rate": 0.041103, "close_rate": 0.04130903007518797, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4329124394813033, "profit_abs": 1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-25 15:45:00+00:00", "close_date": "2018-01-25 16:15:00+00:00", "trade_duration": 30, "open_rate": 5.428e-05, "close_rate": 5.509624060150376e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1842.2991893883568, "profit_abs": 0.0010000000000000148}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 17:45:00+00:00", "close_date": "2018-01-25 23:15:00+00:00", "trade_duration": 330, "open_rate": 5.414e-05, "close_rate": 5.441137844611528e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1847.063169560399, "profit_abs": -1.3877787807814457e-17}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-25 21:15:00+00:00", "close_date": "2018-01-25 21:55:00+00:00", "trade_duration": 40, "open_rate": 0.04140777, "close_rate": 0.0416153277443609, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.415005686130888, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.0, "open_date": "2018-01-26 02:05:00+00:00", "close_date": "2018-01-26 02:45:00+00:00", "trade_duration": 40, "open_rate": 0.00254309, "close_rate": 0.002555837318295739, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 39.32224183965177, "profit_abs": 1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.0, "open_date": "2018-01-26 02:55:00+00:00", "close_date": "2018-01-26 15:10:00+00:00", "trade_duration": 735, "open_rate": 5.607e-05, "close_rate": 5.6351052631578935e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1783.4849295523454, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETC/BTC", "profit_percent": 0.0, "open_date": "2018-01-26 06:10:00+00:00", "close_date": "2018-01-26 09:25:00+00:00", "trade_duration": 195, "open_rate": 0.00253806, "close_rate": 0.0025507821052631577, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 39.400171784748984, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.0, "open_date": "2018-01-26 07:25:00+00:00", "close_date": "2018-01-26 09:55:00+00:00", "trade_duration": 150, "open_rate": 0.0415, "close_rate": 0.04170802005012531, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4096385542168677, "profit_abs": 0.0}, {"pair": "XLM/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-26 09:55:00+00:00", "close_date": "2018-01-26 10:25:00+00:00", "trade_duration": 30, "open_rate": 5.321e-05, "close_rate": 5.401015037593984e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1879.3459875963165, "profit_abs": 0.000999999999999987}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-26 16:05:00+00:00", "close_date": "2018-01-26 16:45:00+00:00", "trade_duration": 40, "open_rate": 0.02772046, "close_rate": 0.02785940967418546, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.6074437437185387, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": 0.0, "open_date": "2018-01-26 23:35:00+00:00", "close_date": "2018-01-27 00:15:00+00:00", "trade_duration": 40, "open_rate": 0.09461341, "close_rate": 0.09508766268170424, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0569326272036914, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 00:35:00+00:00", "close_date": "2018-01-27 01:30:00+00:00", "trade_duration": 55, "open_rate": 5.615e-05, "close_rate": 5.643145363408521e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1780.9439002671415, "profit_abs": -1.3877787807814457e-17}, {"pair": "ADA/BTC", "profit_percent": -0.07877175, "open_date": "2018-01-27 00:45:00+00:00", "close_date": "2018-01-30 04:45:00+00:00", "trade_duration": 4560, "open_rate": 5.556e-05, "close_rate": 5.144e-05, "open_at_end": true, "sell_reason": "force_sell", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1799.8560115190785, "profit_abs": -0.007896868250539965}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 02:30:00+00:00", "close_date": "2018-01-27 11:25:00+00:00", "trade_duration": 535, "open_rate": 0.06900001, "close_rate": 0.06934587471177944, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4492751522789635, "profit_abs": 0.0}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 06:25:00+00:00", "close_date": "2018-01-27 07:05:00+00:00", "trade_duration": 40, "open_rate": 0.09449985, "close_rate": 0.0949735334586466, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.058202737887944, "profit_abs": 0.0}, {"pair": "ZEC/BTC", "profit_percent": -0.04815133, "open_date": "2018-01-27 09:40:00+00:00", "close_date": "2018-01-30 04:40:00+00:00", "trade_duration": 4020, "open_rate": 0.0410697, "close_rate": 0.03928809, "open_at_end": true, "sell_reason": "force_sell", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 2.4348850855983852, "profit_abs": -0.004827170578309559}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 11:45:00+00:00", "close_date": "2018-01-27 12:30:00+00:00", "trade_duration": 45, "open_rate": 0.0285, "close_rate": 0.02864285714285714, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.5087719298245617, "profit_abs": 0.0}, {"pair": "XMR/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 12:35:00+00:00", "close_date": "2018-01-27 15:25:00+00:00", "trade_duration": 170, "open_rate": 0.02866372, "close_rate": 0.02880739779448621, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 3.4887307020861216, "profit_abs": -1.3877787807814457e-17}, {"pair": "ETH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 15:50:00+00:00", "close_date": "2018-01-27 16:50:00+00:00", "trade_duration": 60, "open_rate": 0.095381, "close_rate": 0.09585910025062656, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.0484268355332824, "profit_abs": 1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 17:05:00+00:00", "close_date": "2018-01-27 17:45:00+00:00", "trade_duration": 40, "open_rate": 0.06759092, "close_rate": 0.06792972160401002, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4794886650455417, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": -0.0, "open_date": "2018-01-27 23:40:00+00:00", "close_date": "2018-01-28 01:05:00+00:00", "trade_duration": 85, "open_rate": 0.00258501, "close_rate": 0.002597967443609022, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 38.684569885609726, "profit_abs": -1.3877787807814457e-17}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-28 02:25:00+00:00", "close_date": "2018-01-28 08:10:00+00:00", "trade_duration": 345, "open_rate": 0.06698502, "close_rate": 0.0673207845112782, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4928710926711672, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-28 10:25:00+00:00", "close_date": "2018-01-28 16:30:00+00:00", "trade_duration": 365, "open_rate": 0.0677177, "close_rate": 0.06805713709273183, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4767187899175547, "profit_abs": -1.3877787807814457e-17}, {"pair": "XLM/BTC", "profit_percent": 0.0, "open_date": "2018-01-28 20:35:00+00:00", "close_date": "2018-01-28 21:35:00+00:00", "trade_duration": 60, "open_rate": 5.215e-05, "close_rate": 5.2411403508771925e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1917.5455417066157, "profit_abs": 0.0}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-28 22:00:00+00:00", "close_date": "2018-01-28 22:30:00+00:00", "trade_duration": 30, "open_rate": 0.00273809, "close_rate": 0.002779264285714285, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 36.5218089982433, "profit_abs": 0.0010000000000000009}, {"pair": "ETC/BTC", "profit_percent": 0.00997506, "open_date": "2018-01-29 00:00:00+00:00", "close_date": "2018-01-29 00:30:00+00:00", "trade_duration": 30, "open_rate": 0.00274632, "close_rate": 0.002787618045112782, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 36.412362725392526, "profit_abs": 0.0010000000000000148}, {"pair": "LTC/BTC", "profit_percent": 0.0, "open_date": "2018-01-29 02:15:00+00:00", "close_date": "2018-01-29 03:00:00+00:00", "trade_duration": 45, "open_rate": 0.01622478, "close_rate": 0.016306107218045113, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 6.163411768911504, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 03:05:00+00:00", "close_date": "2018-01-29 03:45:00+00:00", "trade_duration": 40, "open_rate": 0.069, "close_rate": 0.06934586466165413, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4492753623188406, "profit_abs": -1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 05:20:00+00:00", "close_date": "2018-01-29 06:55:00+00:00", "trade_duration": 95, "open_rate": 8.755e-05, "close_rate": 8.798884711779448e-05, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1142.204454597373, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 07:00:00+00:00", "close_date": "2018-01-29 19:25:00+00:00", "trade_duration": 745, "open_rate": 0.06825763, "close_rate": 0.06859977350877192, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4650376815016872, "profit_abs": 0.0}, {"pair": "DASH/BTC", "profit_percent": -0.0, "open_date": "2018-01-29 19:45:00+00:00", "close_date": "2018-01-29 20:25:00+00:00", "trade_duration": 40, "open_rate": 0.06713892, "close_rate": 0.06747545593984962, "open_at_end": false, "sell_reason": "roi", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1.4894490408841845, "profit_abs": -1.3877787807814457e-17}, {"pair": "TRX/BTC", "profit_percent": -0.0199116, "open_date": "2018-01-29 23:30:00+00:00", "close_date": "2018-01-30 04:45:00+00:00", "trade_duration": 315, "open_rate": 8.934e-05, "close_rate": 8.8e-05, "open_at_end": true, "sell_reason": "force_sell", "open_fee": 0.0025, "close_fee": 0.0025, "amount": 1119.3194537721067, "profit_abs": -0.0019961383478844796}], "results_per_pair": [{"key": "TRX/BTC", "trades": 15, "profit_mean": 0.0023467073333333323, "profit_mean_pct": 0.23467073333333321, "profit_sum": 0.035200609999999986, "profit_sum_pct": 3.5200609999999988, "profit_total_abs": 0.0035288616521155086, "profit_total_pct": 1.1733536666666662, "duration_avg": "2:28:00", "wins": 9, "draws": 2, "losses": 4}, {"key": "ADA/BTC", "trades": 29, "profit_mean": -0.0011598141379310352, "profit_mean_pct": -0.11598141379310352, "profit_sum": -0.03363461000000002, "profit_sum_pct": -3.3634610000000023, "profit_total_abs": -0.0033718682505400333, "profit_total_pct": -1.1211536666666675, "duration_avg": "5:35:00", "wins": 9, "draws": 11, "losses": 9}, {"key": "XLM/BTC", "trades": 21, "profit_mean": 0.0026243899999999994, "profit_mean_pct": 0.2624389999999999, "profit_sum": 0.05511218999999999, "profit_sum_pct": 5.511218999999999, "profit_total_abs": 0.005525000000000002, "profit_total_pct": 1.8370729999999995, "duration_avg": "3:21:00", "wins": 12, "draws": 3, "losses": 6}, {"key": "ETH/BTC", "trades": 21, "profit_mean": 0.0009500057142857142, "profit_mean_pct": 0.09500057142857142, "profit_sum": 0.01995012, "profit_sum_pct": 1.9950119999999998, "profit_total_abs": 0.0019999999999999463, "profit_total_pct": 0.6650039999999999, "duration_avg": "2:17:00", "wins": 5, "draws": 10, "losses": 6}, {"key": "XMR/BTC", "trades": 16, "profit_mean": -0.0027899012500000007, "profit_mean_pct": -0.2789901250000001, "profit_sum": -0.04463842000000001, "profit_sum_pct": -4.463842000000001, "profit_total_abs": -0.0044750000000000345, "profit_total_pct": -1.4879473333333337, "duration_avg": "8:41:00", "wins": 6, "draws": 5, "losses": 5}, {"key": "ZEC/BTC", "trades": 21, "profit_mean": -0.00039290904761904774, "profit_mean_pct": -0.03929090476190478, "profit_sum": -0.008251090000000003, "profit_sum_pct": -0.8251090000000003, "profit_total_abs": -0.000827170578309569, "profit_total_pct": -0.27503633333333344, "duration_avg": "4:17:00", "wins": 8, "draws": 7, "losses": 6}, {"key": "NXT/BTC", "trades": 12, "profit_mean": -0.0012261025000000006, "profit_mean_pct": -0.12261025000000006, "profit_sum": -0.014713230000000008, "profit_sum_pct": -1.4713230000000008, "profit_total_abs": -0.0014750000000000874, "profit_total_pct": -0.4904410000000003, "duration_avg": "0:57:00", "wins": 4, "draws": 3, "losses": 5}, {"key": "LTC/BTC", "trades": 8, "profit_mean": 0.00748129625, "profit_mean_pct": 0.748129625, "profit_sum": 0.05985037, "profit_sum_pct": 5.985037, "profit_total_abs": 0.006000000000000019, "profit_total_pct": 1.9950123333333334, "duration_avg": "1:59:00", "wins": 5, "draws": 2, "losses": 1}, {"key": "ETC/BTC", "trades": 20, "profit_mean": 0.0022568569999999997, "profit_mean_pct": 0.22568569999999996, "profit_sum": 0.04513713999999999, "profit_sum_pct": 4.513713999999999, "profit_total_abs": 0.004525000000000001, "profit_total_pct": 1.504571333333333, "duration_avg": "1:45:00", "wins": 11, "draws": 4, "losses": 5}, {"key": "DASH/BTC", "trades": 16, "profit_mean": 0.0018703237499999997, "profit_mean_pct": 0.18703237499999997, "profit_sum": 0.029925179999999996, "profit_sum_pct": 2.9925179999999996, "profit_total_abs": 0.002999999999999961, "profit_total_pct": 0.9975059999999999, "duration_avg": "3:03:00", "wins": 4, "draws": 7, "losses": 5}, {"key": "TOTAL", "trades": 179, "profit_mean": 0.0008041243575418989, "profit_mean_pct": 0.0804124357541899, "profit_sum": 0.1439382599999999, "profit_sum_pct": 14.39382599999999, "profit_total_abs": 0.014429822823265714, "profit_total_pct": 4.797941999999996, "duration_avg": "3:40:00", "wins": 73, "draws": 54, "losses": 52}], "sell_reason_summary": [{"sell_reason": "roi", "trades": 170, "wins": 73, "draws": 54, "losses": 43, "profit_mean": 0.005398268352941177, "profit_mean_pct": 0.54, "profit_sum": 0.91770562, "profit_sum_pct": 91.77, "profit_total_abs": 0.09199999999999964, "profit_pct_total": 30.59}, {"sell_reason": "stop_loss", "trades": 6, "wins": 0, "draws": 0, "losses": 6, "profit_mean": -0.10448878000000002, "profit_mean_pct": -10.45, "profit_sum": -0.6269326800000001, "profit_sum_pct": -62.69, "profit_total_abs": -0.06284999999999992, "profit_pct_total": -20.9}, {"sell_reason": "force_sell", "trades": 3, "wins": 0, "draws": 0, "losses": 3, "profit_mean": -0.04894489333333333, "profit_mean_pct": -4.89, "profit_sum": -0.14683468, "profit_sum_pct": -14.68, "profit_total_abs": -0.014720177176734003, "profit_pct_total": -4.89}], "left_open_trades": [{"key": "TRX/BTC", "trades": 1, "profit_mean": -0.0199116, "profit_mean_pct": -1.9911600000000003, "profit_sum": -0.0199116, "profit_sum_pct": -1.9911600000000003, "profit_total_abs": -0.0019961383478844796, "profit_total_pct": -0.6637200000000001, "duration_avg": "5:15:00", "wins": 0, "draws": 0, "losses": 1}, {"key": "ADA/BTC", "trades": 1, "profit_mean": -0.07877175, "profit_mean_pct": -7.877175, "profit_sum": -0.07877175, "profit_sum_pct": -7.877175, "profit_total_abs": -0.007896868250539965, "profit_total_pct": -2.625725, "duration_avg": "3 days, 4:00:00", "wins": 0, "draws": 0, "losses": 1}, {"key": "ZEC/BTC", "trades": 1, "profit_mean": -0.04815133, "profit_mean_pct": -4.815133, "profit_sum": -0.04815133, "profit_sum_pct": -4.815133, "profit_total_abs": -0.004827170578309559, "profit_total_pct": -1.6050443333333335, "duration_avg": "2 days, 19:00:00", "wins": 0, "draws": 0, "losses": 1}, {"key": "TOTAL", "trades": 3, "profit_mean": -0.04894489333333333, "profit_mean_pct": -4.894489333333333, "profit_sum": -0.14683468, "profit_sum_pct": -14.683468, "profit_total_abs": -0.014720177176734003, "profit_total_pct": -4.8944893333333335, "duration_avg": "2 days, 1:25:00", "wins": 0, "draws": 0, "losses": 3}], "total_trades": 179, "backtest_start": "2018-01-30 04:45:00+00:00", "backtest_start_ts": 1517287500, "backtest_end": "2018-01-30 04:45:00+00:00", "backtest_end_ts": 1517287500, "backtest_days": 0, "trades_per_day": null, "market_change": 0.25, "stake_amount": 0.1, "max_drawdown": 0.21142322000000008, "drawdown_start": "2018-01-24 14:25:00+00:00", "drawdown_start_ts": 1516803900.0, "drawdown_end": "2018-01-30 04:45:00+00:00", "drawdown_end_ts": 1517287500.0, "pairlist": ["TRX/BTC", "ADA/BTC", "XLM/BTC", "ETH/BTC", "XMR/BTC", "ZEC/BTC","NXT/BTC", "LTC/BTC", "ETC/BTC", "DASH/BTC"]}}, "strategy_comparison": [{"key": "DefaultStrategy", "trades": 179, "profit_mean": 0.0008041243575418989, "profit_mean_pct": 0.0804124357541899, "profit_sum": 0.1439382599999999, "profit_sum_pct": 14.39382599999999, "profit_total_abs": 0.014429822823265714, "profit_total_pct": 4.797941999999996, "duration_avg": "3:40:00", "wins": 73, "draws": 54, "losses": 52}]}