diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6a111944..42668e46f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,8 @@ on: - develop - github_actions_tests tags: + release: + types: [published] pull_request: schedule: - cron: '0 5 * * 4' @@ -18,10 +20,10 @@ jobs: strategy: matrix: os: [ ubuntu-18.04, macos-latest ] - python-version: [3.7] + python-version: [3.7, 3.8] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 @@ -64,19 +66,17 @@ jobs: pip install -e . - name: Tests - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - COVERALLS_SERVICE_NAME: travis-ci - TRAVIS: "true" run: | pytest --random-order --cov=freqtrade --cov-config=.coveragerc + + - name: Coveralls + if: (startsWith(matrix.os, 'ubuntu') && matrix.python-version == '3.8') + env: + # Coveralls token. Not used as secret due to github not providing secrets to forked repositories + COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu + run: | # Allow failure for coveralls - # Fake travis environment to get coveralls working correctly - export TRAVIS_PULL_REQUEST="https://github.com/${GITHUB_REPOSITORY}/pull/$(cat $GITHUB_EVENT_PATH | jq -r .number)" - export TRAVIS_BRANCH=${GITHUB_REF#"ref/heads"} - export CI_BRANCH=${GITHUB_REF#"ref/heads"} - echo "${TRAVIS_BRANCH}" - coveralls || true + coveralls -v || true - name: Backtesting run: | @@ -115,10 +115,10 @@ jobs: strategy: matrix: os: [ windows-latest ] - python-version: [3.7] + python-version: [3.7, 3.8] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 @@ -130,8 +130,7 @@ jobs: if: startsWith(runner.os, 'Windows') with: path: ~\AppData\Local\pip\Cache - key: ${{ runner.os }}-pip - restore-keys: ${{ runner.os }}-pip + key: ${{ matrix.os }}-${{ matrix.python-version }}-pip - name: Installation run: | @@ -175,7 +174,7 @@ jobs: docs_check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Documentation syntax run: | @@ -193,15 +192,40 @@ jobs: deploy: needs: [ build, build_windows, docs_check ] runs-on: ubuntu-18.04 - if: (github.event_name == 'push' || github.event_name == 'schedule') && github.repository == 'freqtrade/freqtrade' + if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 - name: Extract branch name shell: bash run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" id: extract_branch + - name: Build distribution + run: | + pip install -U setuptools wheel + python setup.py sdist bdist_wheel + + - name: Publish to PyPI (Test) + uses: pypa/gh-action-pypi-publish@master + if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release') + with: + user: __token__ + password: ${{ secrets.pypi_test_password }} + repository_url: https://test.pypi.org/legacy/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@master + if: (steps.extract_branch.outputs.branch == 'master' || github.event_name == 'release') + with: + user: __token__ + password: ${{ secrets.pypi_password }} + - name: Build and test and push docker image env: IMAGE_NAME: freqtradeorg/freqtrade diff --git a/.gitignore b/.gitignore index 9ac2c9d5d..f206fce66 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ user_data/* !user_data/strategy/sample_strategy.py !user_data/notebooks user_data/notebooks/* -!user_data/notebooks/*example.ipynb freqtrade-plot.html freqtrade-profit-plot.html diff --git a/.travis.yml b/.travis.yml index ec688a1f4..0cb76b78b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: true os: - linux dist: xenial @@ -11,10 +10,10 @@ env: global: - IMAGE_NAME=freqtradeorg/freqtrade install: -- cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. +- cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies; cd .. - export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH - export TA_LIBRARY_PATH=${HOME}/dependencies/lib -- export TA_INCLUDE_PATH=${HOME}/dependencies/lib/include +- export TA_INCLUDE_PATH=${HOME}/dependencies/include - pip install -r requirements-dev.txt - pip install -e . jobs: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a087103c6..90594866a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ pytest tests/test_.py::test_ #### Run Flake8 ```bash -flake8 freqtrade +flake8 freqtrade tests scripts ``` We receive a lot of code that fails the `flake8` checks. @@ -109,11 +109,11 @@ Exceptions: Contributors may be given commit privileges. Preference will be given to those with: -1. Past contributions to FreqTrade and other related open-source projects. Contributions to FreqTrade include both code (both accepted and pending) and friendly participation in the issue tracker and Pull request reviews. Quantity and quality are considered. +1. Past contributions to Freqtrade and other related open-source projects. Contributions to Freqtrade include both code (both accepted and pending) and friendly participation in the issue tracker and Pull request reviews. Quantity and quality are considered. 1. A coding style that the other core committers find simple, minimal, and clean. 1. Access to resources for cross-platform development and testing. 1. Time to devote to the project regularly. -Being a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust FreqTrade with their Exchange API keys). +Being a Committer does not grant write permission on `develop` or `master` for security reasons (Users trust Freqtrade with their Exchange API keys). After being Committer for some time, a Committer may be named Core Committer and given full repository access. diff --git a/Dockerfile b/Dockerfile index dc9b04403..b6333fb13 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7.5-slim-stretch +FROM python:3.8.3-slim-buster RUN apt-get update \ && apt-get -y install curl build-essential libssl-dev \ diff --git a/MANIFEST.in b/MANIFEST.in index 7529152a0..c67f5258f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ include LICENSE include README.md include config.json.example recursive-include freqtrade *.py +recursive-include freqtrade/templates/ *.j2 *.ipynb diff --git a/README.md b/README.md index a1feeab67..88070d45e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Freqtrade -[![Build Status](https://travis-ci.org/freqtrade/freqtrade.svg?branch=develop)](https://travis-ci.org/freqtrade/freqtrade) +[![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/) [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) [![Documentation](https://readthedocs.org/projects/freqtrade/badge/)](https://www.freqtrade.io) [![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability) @@ -25,7 +25,8 @@ hesitate to read the source code and understand the mechanism of this bot. ## Exchange marketplaces supported - [X] [Bittrex](https://bittrex.com/) -- [X] [Binance](https://www.binance.com/) ([*Note for binance users](#a-note-on-binance)) +- [X] [Binance](https://www.binance.com/) ([*Note for binance users](docs/exchanges.md#blacklists)) +- [X] [Kraken](https://kraken.com/) - [ ] [113 others to tests](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ## Documentation diff --git a/bin/freqtrade b/bin/freqtrade deleted file mode 100755 index 25c94fe98..000000000 --- a/bin/freqtrade +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import warnings - -from freqtrade.main import main - -warnings.warn( - "Deprecated - To continue to run the bot like this, please run `pip install -e .` again.", - DeprecationWarning) -main(sys.argv[1:]) diff --git a/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl deleted file mode 100644 index 87469a199..000000000 Binary files a/build_helpers/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl new file mode 100644 index 000000000..bd61e812b Binary files /dev/null and b/build_helpers/TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl new file mode 100644 index 000000000..f81addb44 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.18-cp38-cp38-win_amd64.whl differ diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 index 30427c3cc..0a55b6ddd 100644 --- a/build_helpers/install_windows.ps1 +++ b/build_helpers/install_windows.ps1 @@ -2,7 +2,16 @@ # Downloaded from https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib # Invoke-WebRequest -Uri "https://download.lfd.uci.edu/pythonlibs/xxxxxxx/TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" -OutFile "TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl" -pip install build_helpers\TA_Lib-0.4.17-cp37-cp37m-win_amd64.whl +python -m pip install --upgrade pip + +$pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" + +if ($pyv -eq '3.7') { + pip install build_helpers\TA_Lib-0.4.18-cp37-cp37m-win_amd64.whl +} +if ($pyv -eq '3.8') { + pip install build_helpers\TA_Lib-0.4.18-cp38-cp38-win_amd64.whl +} pip install -r requirements-dev.txt pip install -e . diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index 17d5230c9..013644563 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -23,7 +23,7 @@ if [ $? -ne 0 ]; then fi # Run backtest -docker run --rm -v $(pwd)/config.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy DefaultStrategy +docker run --rm -v $(pwd)/config.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy if [ $? -ne 0 ]; then echo "failed running backtest" diff --git a/config.json.example b/config.json.example index a2add358f..d37a6b336 100644 --- a/config.json.example +++ b/config.json.example @@ -2,9 +2,11 @@ "max_open_trades": 3, "stake_currency": "BTC", "stake_amount": 0.05, + "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", - "ticker_interval" : "5m", + "ticker_interval": "5m", "dry_run": false, + "cancel_open_orders_on_exit": false, "trailing_stop": false, "unfilledtimeout": { "buy": 10, @@ -22,7 +24,7 @@ "ask_strategy":{ "use_order_book": false, "order_book_min": 1, - "order_book_max": 9, + "order_book_max": 1, "use_sell_signal": true, "sell_profit_only": false, "ignore_roi_if_buy_signal": false @@ -43,7 +45,7 @@ "DASH/BTC", "ZEC/BTC", "XLM/BTC", - "NXT/BTC", + "XRP/BTC", "TRX/BTC", "ADA/BTC", "XMR/BTC" @@ -59,7 +61,6 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 7, - "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, diff --git a/config_binance.json.example b/config_binance.json.example index 8dc6f5c3a..5d7b6b656 100644 --- a/config_binance.json.example +++ b/config_binance.json.example @@ -2,9 +2,11 @@ "max_open_trades": 3, "stake_currency": "BTC", "stake_amount": 0.05, + "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", - "ticker_interval" : "5m", + "ticker_interval": "5m", "dry_run": true, + "cancel_open_orders_on_exit": false, "trailing_stop": false, "unfilledtimeout": { "buy": 10, @@ -22,7 +24,7 @@ "ask_strategy":{ "use_order_book": false, "order_book_min": 1, - "order_book_max": 9, + "order_book_max": 1, "use_sell_signal": true, "sell_profit_only": false, "ignore_roi_if_buy_signal": false @@ -64,7 +66,6 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 7, - "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, diff --git a/config_full.json.example b/config_full.json.example index b9631f63d..0cd265cbe 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -2,9 +2,13 @@ "max_open_trades": 3, "stake_currency": "BTC", "stake_amount": 0.05, + "tradable_balance_ratio": 0.99, "fiat_display_currency": "USD", - "amount_reserve_percent" : 0.05, + "amount_reserve_percent": 0.05, + "amend_last_stake_amount": false, + "last_stake_amount_min_ratio": 0.5, "dry_run": false, + "cancel_open_orders_on_exit": false, "ticker_interval": "5m", "trailing_stop": false, "trailing_stop_positive": 0.005, @@ -22,6 +26,7 @@ "sell": 30 }, "bid_strategy": { + "price_side": "bid", "use_order_book": false, "ask_last_balance": 0.0, "order_book_top": 1, @@ -31,9 +36,10 @@ } }, "ask_strategy":{ + "price_side": "ask", "use_order_book": false, "order_book_min": 1, - "order_book_max": 9, + "order_book_max": 1, "use_sell_signal": true, "sell_profit_only": false, "ignore_roi_if_buy_signal": false @@ -59,8 +65,8 @@ "refresh_period": 1800 }, {"method": "PrecisionFilter"}, - {"method": "PriceFilter", "low_price_ratio": 0.01 - } + {"method": "PriceFilter", "low_price_ratio": 0.01}, + {"method": "SpreadFilter", "max_spread_ratio": 0.005} ], "exchange": { "name": "bittrex", @@ -96,7 +102,6 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 7, - "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, @@ -116,6 +121,7 @@ "enabled": false, "listen_ip_address": "127.0.0.1", "listen_port": 8080, + "jwt_secret_key": "somethingrandom", "username": "freqtrader", "password": "SuperSecurePassword" }, @@ -127,5 +133,7 @@ "heartbeat_interval": 60 }, "strategy": "DefaultStrategy", - "strategy_path": "user_data/strategies/" + "strategy_path": "user_data/strategies/", + "dataformat_ohlcv": "json", + "dataformat_trades": "jsongz" } diff --git a/config_kraken.json.example b/config_kraken.json.example index 401ee7c9f..54fbf4a00 100644 --- a/config_kraken.json.example +++ b/config_kraken.json.example @@ -2,9 +2,11 @@ "max_open_trades": 5, "stake_currency": "EUR", "stake_amount": 10, + "tradable_balance_ratio": 0.99, "fiat_display_currency": "EUR", - "ticker_interval" : "5m", + "ticker_interval": "5m", "dry_run": true, + "cancel_open_orders_on_exit": false, "trailing_stop": false, "unfilledtimeout": { "buy": 10, @@ -22,7 +24,7 @@ "ask_strategy":{ "use_order_book": false, "order_book_min": 1, - "order_book_max": 9, + "order_book_max": 1, "use_sell_signal": true, "sell_profit_only": false, "ignore_roi_if_buy_signal": false @@ -70,7 +72,6 @@ "enabled": false, "process_throttle_secs": 3600, "calculate_since_number_of_days": 7, - "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, diff --git a/docker-compose.yml b/docker-compose.yml index cae98c3ee..49d83aa5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,19 @@ version: '3' services: freqtrade: image: freqtradeorg/freqtrade:master + # image: freqtradeorg/freqtrade:develop + # Build step - only needed when additional dependencies are needed + # build: + # context: . + # dockerfile: "./Dockerfile.technical" + restart: unless-stopped + container_name: freqtrade volumes: - "./user_data:/freqtrade/user_data" - - "./config.json:/freqtrade/config.json" + # Default command used when running `docker compose up` + command: > + trade + --logfile /freqtrade/user_data/logs/freqtrade.log + --db-url sqlite:////freqtrade/user_data/tradesv3.sqlite + --config /freqtrade/user_data/config.json + --strategy SampleStrategy diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 20af0aaab..25b4bd900 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -4,6 +4,34 @@ This page explains some advanced Hyperopt topics that may require higher coding skills and Python knowledge than creation of an ordinal hyperoptimization class. +## Derived hyperopt classes + +Custom hyperop classes can be derived in the same way [it can be done for strategies](strategy-customization.md#derived-strategies). + +Applying to hyperoptimization, as an example, you may override how dimensions are defined in your optimization hyperspace: + +```python +class MyAwesomeHyperOpt(IHyperOpt): + ... + # Uses default stoploss dimension + +class MyAwesomeHyperOpt2(MyAwesomeHyperOpt): + @staticmethod + def stoploss_space() -> List[Dimension]: + # Override boundaries for stoploss + return [ + Real(-0.33, -0.01, name='stoploss'), + ] +``` + +and then quickly switch between hyperopt classes, running optimization process with hyperopt class you need in each particular case: + +``` +$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt ... +or +$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt2 ... +``` + ## Creating and using a custom loss function To use a custom loss function class, make sure that the function `hyperopt_loss_function` is defined in your custom hyperopt loss class. diff --git a/docs/advanced-setup.md b/docs/advanced-setup.md index 2d3fe36f5..95480a2c6 100644 --- a/docs/advanced-setup.md +++ b/docs/advanced-setup.md @@ -37,30 +37,30 @@ as the watchdog. ## Advanced Logging -On many Linux systems the bot can be configured to send its log messages to `syslog` or `journald` system services. Logging to a remote `syslog` server is also available on Windows. The special values for the `--logfilename` command line option can be used for this. +On many Linux systems the bot can be configured to send its log messages to `syslog` or `journald` system services. Logging to a remote `syslog` server is also available on Windows. The special values for the `--logfile` command line option can be used for this. ### Logging to syslog -To send Freqtrade log messages to a local or remote `syslog` service use the `--logfilename` command line option with the value in the following format: +To send Freqtrade log messages to a local or remote `syslog` service use the `--logfile` command line option with the value in the following format: -* `--logfilename syslog:` -- send log messages to `syslog` service using the `` as the syslog address. +* `--logfile syslog:` -- send log messages to `syslog` service using the `` as the syslog address. The syslog address can be either a Unix domain socket (socket filename) or a UDP socket specification, consisting of IP address and UDP port, separated by the `:` character. So, the following are the examples of possible usages: -* `--logfilename syslog:/dev/log` -- log to syslog (rsyslog) using the `/dev/log` socket, suitable for most systems. -* `--logfilename syslog` -- same as above, the shortcut for `/dev/log`. -* `--logfilename syslog:/var/run/syslog` -- log to syslog (rsyslog) using the `/var/run/syslog` socket. Use this on MacOS. -* `--logfilename syslog:localhost:514` -- log to local syslog using UDP socket, if it listens on port 514. -* `--logfilename syslog::514` -- log to remote syslog at IP address and port 514. This may be used on Windows for remote logging to an external syslog server. +* `--logfile syslog:/dev/log` -- log to syslog (rsyslog) using the `/dev/log` socket, suitable for most systems. +* `--logfile syslog` -- same as above, the shortcut for `/dev/log`. +* `--logfile syslog:/var/run/syslog` -- log to syslog (rsyslog) using the `/var/run/syslog` socket. Use this on MacOS. +* `--logfile syslog:localhost:514` -- log to local syslog using UDP socket, if it listens on port 514. +* `--logfile syslog::514` -- log to remote syslog at IP address and port 514. This may be used on Windows for remote logging to an external syslog server. Log messages are send to `syslog` with the `user` facility. So you can see them with the following commands: * `tail -f /var/log/user`, or * install a comprehensive graphical viewer (for instance, 'Log File Viewer' for Ubuntu). -On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. +On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfile syslog` or `--logfile journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. For `rsyslog` the messages from the bot can be redirected into a separate dedicated log file. To achieve this, add ``` @@ -78,9 +78,9 @@ $RepeatedMsgReduction on This needs the `systemd` python package installed as the dependency, which is not available on Windows. Hence, the whole journald logging functionality is not available for a bot running on Windows. -To send Freqtrade log messages to `journald` system service use the `--logfilename` command line option with the value in the following format: +To send Freqtrade log messages to `journald` system service use the `--logfile` command line option with the value in the following format: -* `--logfilename journald` -- send log messages to `journald`. +* `--logfile journald` -- send log messages to `journald`. Log messages are send to `journald` with the `user` facility. So you can see them with the following commands: @@ -89,4 +89,4 @@ Log messages are send to `journald` with the `user` facility. So you can see the There are many other options in the `journalctl` utility to filter the messages, see manual pages for this utility. -On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfilename syslog` or `--logfilename journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. +On many systems `syslog` (`rsyslog`) fetches data from `journald` (and vice versa), so both `--logfile syslog` or `--logfile journald` can be used and the messages be viewed with both `journalctl` and a syslog viewer utility. You can combine this in any way which suites you better. diff --git a/docs/assets/plot-dataframe.png b/docs/assets/plot-dataframe.png index eb90a1734..6310b23b4 100644 Binary files a/docs/assets/plot-dataframe.png and b/docs/assets/plot-dataframe.png differ diff --git a/docs/assets/plot-dataframe2.png b/docs/assets/plot-dataframe2.png new file mode 100644 index 000000000..d744b2035 Binary files /dev/null and b/docs/assets/plot-dataframe2.png differ diff --git a/docs/backtesting.md b/docs/backtesting.md index 017289905..9b2997510 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -11,30 +11,34 @@ Now you have good Buy and Sell strategies and some historic data, you want to te real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). -Backtesting will use the crypto-currencies (pairs) from your config file and load ticker data from `user_data/data/` by default. -If no data is available for the exchange / pair / ticker interval combination, backtesting will ask you to download them first using `freqtrade download-data`. +Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHCLV) data from `user_data/data/` by default. +If no data is available for the exchange / pair / timeframe (ticker interval) combination, backtesting will ask you to download them first using `freqtrade download-data`. For details on downloading, please refer to the [Data Downloading](data-download.md) section in the documentation. The result of backtesting will confirm if your bot has better odds of making a profit than a loss. -!!! Tip "Using dynamic pairlists for backtesting" - While using dynamic pairlists during backtesting is not possible, a dynamic pairlist using current data can be generated via the [`test-pairlist`](utils.md#test-pairlist) command, and needs to be specified as `"pair_whitelist"` attribute in the configuration. +!!! Warning "Using dynamic pairlists for backtesting" + Using dynamic pairlists is possible, however it relies on the current market conditions - which will not reflect the historic status of the pairlist. + Also, when using pairlists other than StaticPairlist, reproducability of backtesting-results cannot be guaranteed. + Please read the [pairlists documentation](configuration.md#pairlists) for more information. + + To achieve reproducible results, best generate a pairlist via the [`test-pairlist`](utils.md#test-pairlist) command and use that as static pairlist. ### Run a backtesting against the currencies listed in your config file -#### With 5 min tickers (Per default) +#### With 5 min candle (OHLCV) data (per default) ```bash freqtrade backtesting ``` -#### With 1 min tickers +#### With 1 min candle (OHLCV) data ```bash freqtrade backtesting --ticker-interval 1m ``` -#### Using a different on-disk ticker-data source +#### Using a different on-disk historical candle (OHLCV) data source Assume you downloaded the history data from the Bittrex exchange and kept it in the `user_data/data/bittrex-20180101` directory. You can then use this data for backtesting as follows: @@ -78,13 +82,17 @@ Please also read about the [strategy startup period](strategy-customization.md#s #### Supplying custom fee value Sometimes your account has certain fee rebates (fee reductions starting with a certain account size or monthly volume), which are not visible to ccxt. -To account for this in backtesting, you can use `--fee 0.001` to supply this value to backtesting. -This fee must be a percentage, and will be applied twice (once for trade entry, and once for trade exit). +To account for this in backtesting, you can use the `--fee` command line option to supply this value to backtesting. +This fee must be a ratio, and will be applied twice (once for trade entry, and once for trade exit). + +For example, if the buying and selling commission fee is 0.1% (i.e., 0.001 written as ratio), then you would run backtesting as the following: ```bash freqtrade backtesting --fee 0.001 ``` +!!! Note + Only supply this option (or the corresponding configuration parameter) if you want to experiment with different fee values. By default, Backtesting fetches the default fee from the exchange pair/market info. #### Running backtest with smaller testset by using timerange @@ -115,45 +123,46 @@ A backtesting result will look like that: ``` ========================================================= BACKTESTING REPORT ======================================================== -| pair | buy count | avg profit % | cum profit % | tot profit BTC | tot profit % | avg duration | profit | loss | -|:---------|------------:|---------------:|---------------:|-----------------:|---------------:|:---------------|---------:|-------:| -| ADA/BTC | 35 | -0.11 | -3.88 | -0.00019428 | -1.94 | 4:35:00 | 14 | 21 | -| ARK/BTC | 11 | -0.41 | -4.52 | -0.00022647 | -2.26 | 2:03:00 | 3 | 8 | -| BTS/BTC | 32 | 0.31 | 9.78 | 0.00048938 | 4.89 | 5:05:00 | 18 | 14 | -| DASH/BTC | 13 | -0.08 | -1.07 | -0.00005343 | -0.53 | 4:39:00 | 6 | 7 | -| ENG/BTC | 18 | 1.36 | 24.54 | 0.00122807 | 12.27 | 2:50:00 | 8 | 10 | -| EOS/BTC | 36 | 0.08 | 3.06 | 0.00015304 | 1.53 | 3:34:00 | 16 | 20 | -| ETC/BTC | 26 | 0.37 | 9.51 | 0.00047576 | 4.75 | 6:14:00 | 11 | 15 | -| ETH/BTC | 33 | 0.30 | 9.96 | 0.00049856 | 4.98 | 7:31:00 | 16 | 17 | -| IOTA/BTC | 32 | 0.03 | 1.09 | 0.00005444 | 0.54 | 3:12:00 | 14 | 18 | -| LSK/BTC | 15 | 1.75 | 26.26 | 0.00131413 | 13.13 | 2:58:00 | 6 | 9 | -| LTC/BTC | 32 | -0.04 | -1.38 | -0.00006886 | -0.69 | 4:49:00 | 11 | 21 | -| NANO/BTC | 17 | 1.26 | 21.39 | 0.00107058 | 10.70 | 1:55:00 | 10 | 7 | -| NEO/BTC | 23 | 0.82 | 18.97 | 0.00094936 | 9.48 | 2:59:00 | 10 | 13 | -| REQ/BTC | 9 | 1.17 | 10.54 | 0.00052734 | 5.27 | 3:47:00 | 4 | 5 | -| XLM/BTC | 16 | 1.22 | 19.54 | 0.00097800 | 9.77 | 3:15:00 | 7 | 9 | -| XMR/BTC | 23 | -0.18 | -4.13 | -0.00020696 | -2.07 | 5:30:00 | 12 | 11 | -| XRP/BTC | 35 | 0.66 | 22.96 | 0.00114897 | 11.48 | 3:49:00 | 12 | 23 | -| ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 | 15 | -| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 243 | +| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | +|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:| +| ADA/BTC | 35 | -0.11 | -3.88 | -0.00019428 | -1.94 | 4:35:00 | 14 | 0 | 21 | +| ARK/BTC | 11 | -0.41 | -4.52 | -0.00022647 | -2.26 | 2:03:00 | 3 | 0 | 8 | +| BTS/BTC | 32 | 0.31 | 9.78 | 0.00048938 | 4.89 | 5:05:00 | 18 | 0 | 14 | +| DASH/BTC | 13 | -0.08 | -1.07 | -0.00005343 | -0.53 | 4:39:00 | 6 | 0 | 7 | +| ENG/BTC | 18 | 1.36 | 24.54 | 0.00122807 | 12.27 | 2:50:00 | 8 | 0 | 10 | +| EOS/BTC | 36 | 0.08 | 3.06 | 0.00015304 | 1.53 | 3:34:00 | 16 | 0 | 20 | +| ETC/BTC | 26 | 0.37 | 9.51 | 0.00047576 | 4.75 | 6:14:00 | 11 | 0 | 15 | +| ETH/BTC | 33 | 0.30 | 9.96 | 0.00049856 | 4.98 | 7:31:00 | 16 | 0 | 17 | +| IOTA/BTC | 32 | 0.03 | 1.09 | 0.00005444 | 0.54 | 3:12:00 | 14 | 0 | 18 | +| LSK/BTC | 15 | 1.75 | 26.26 | 0.00131413 | 13.13 | 2:58:00 | 6 | 0 | 9 | +| LTC/BTC | 32 | -0.04 | -1.38 | -0.00006886 | -0.69 | 4:49:00 | 11 | 0 | 21 | +| NANO/BTC | 17 | 1.26 | 21.39 | 0.00107058 | 10.70 | 1:55:00 | 10 | 0 | 7 | +| NEO/BTC | 23 | 0.82 | 18.97 | 0.00094936 | 9.48 | 2:59:00 | 10 | 0 | 13 | +| REQ/BTC | 9 | 1.17 | 10.54 | 0.00052734 | 5.27 | 3:47:00 | 4 | 0 | 5 | +| XLM/BTC | 16 | 1.22 | 19.54 | 0.00097800 | 9.77 | 3:15:00 | 7 | 0 | 9 | +| XMR/BTC | 23 | -0.18 | -4.13 | -0.00020696 | -2.07 | 5:30:00 | 12 | 0 | 11 | +| XRP/BTC | 35 | 0.66 | 22.96 | 0.00114897 | 11.48 | 3:49:00 | 12 | 0 | 23 | +| ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 | 0 | 15 | +| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 | ========================================================= SELL REASON STATS ========================================================= -| Sell Reason | Count | -|:-------------------|--------:| -| trailing_stop_loss | 205 | -| stop_loss | 166 | -| sell_signal | 56 | -| force_sell | 2 | +| Sell Reason | Sells | Wins | Draws | Losses | +|:-------------------|--------:|------:|-------:|--------:| +| trailing_stop_loss | 205 | 150 | 0 | 55 | +| stop_loss | 166 | 0 | 0 | 166 | +| sell_signal | 56 | 36 | 0 | 20 | +| force_sell | 2 | 0 | 0 | 2 | ====================================================== LEFT OPEN TRADES REPORT ====================================================== -| pair | buy count | avg profit % | cum profit % | tot profit BTC | tot profit % | avg duration | profit | loss | -|:---------|------------:|---------------:|---------------:|-----------------:|---------------:|:---------------|---------:|-------:| -| ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 | 0 | -| LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 | 0 | -| TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 | 0 | +| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | +|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:| +| ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 | 0 | 0 | +| LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 | 0 | 0 | +| TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 | 0 | 0 | ``` The 1st table contains all trades the bot made, including "left open trades". The 2nd table contains a recap of sell reasons. +This table can tell you which area needs some additional work (i.e. all `sell_signal` trades are losses, so we should disable the sell-signal or work on improving that). The 3rd table contains all trades the bot had to `forcesell` at the end of the backtest period to present a full picture. This is necessary to simulate realistic behaviour, since the backtest period has to end at some point, while realistically, you could leave the bot running forever. @@ -193,7 +202,7 @@ Since backtesting lacks some detailed information about what happens within a ca - Buys happen at open-price - Sell signal sells happen at open-price of the following candle -- Low happens before high for stoploss, protecting capital first. +- Low happens before high for stoploss, protecting capital first - ROI - sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%) - sells are never "below the candle", so a ROI of 2% may result in a sell at 2.4% if low was at 2.4% profit @@ -203,6 +212,7 @@ Since backtesting lacks some detailed information about what happens within a ca - High happens first - adjusting stoploss - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly) - Sell-reason does not explain if a trade was positive or negative, just what triggered the sell (this can look odd if negative ROI values are used) +- Stoploss (and trailing stoploss) is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` and/or `trailing_stop` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes. Taking these assumptions, backtesting tries to mirror real trading as closely as possible. However, backtesting will **never** replace running a strategy in dry-run mode. Also, keep in mind that past results don't guarantee future success. @@ -218,7 +228,7 @@ You can then load the trades to perform further analysis as shown in our [data a To compare multiple strategies, a list of Strategies can be provided to backtesting. -This is limited to 1 ticker-interval per run, however, data is only loaded once from disk so if you have multiple +This is limited to 1 timeframe (ticker interval) value per run. However, data is only loaded once from disk so if you have multiple strategies you'd like to compare, this will give a nice runtime boost. All listed Strategies need to be in the same directory. @@ -232,11 +242,11 @@ There will be an additional table comparing win/losses of the different strategi Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy. ``` -=========================================================== Strategy Summary =========================================================== -| Strategy | buy count | avg profit % | cum profit % | tot profit BTC | tot profit % | avg duration | profit | loss | -|:------------|------------:|---------------:|---------------:|-----------------:|---------------:|:---------------|---------:|-------:| -| Strategy1 | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 243 | -| Strategy2 | 1487 | -0.13 | -197.58 | -0.00988917 | -98.79 | 4:43:00 | 662 | 825 | +=========================================================== STRATEGY SUMMARY =========================================================== +| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | +|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:| +| Strategy1 | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 | +| Strategy2 | 1487 | -0.13 | -197.58 | -0.00988917 | -98.79 | 4:43:00 | 662 | 0 | 825 | ``` ## Next step diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 25818aea6..b1649374a 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -45,19 +45,23 @@ optional arguments: -h, --help show this help message and exit --db-url PATH Override trades database URL, this is useful in custom deployments (default: `sqlite:///tradesv3.sqlite` for - Live Run mode, `sqlite://` for Dry Run). + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). --sd-notify Notify systemd service manager. --dry-run Enforce dry-run for trading (removes Exchange secrets and simulates trades). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -68,6 +72,8 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. +. + ``` ### How to specify which configuration file be used? @@ -138,10 +144,10 @@ It is recommended to use version control to keep track of changes to your strate ### How to use **--strategy**? This parameter will allow you to load your custom strategy class. -Per default without `--strategy` or `-s` the bot will load the -`DefaultStrategy` included with the bot (`freqtrade/strategy/default_strategy.py`). +To test the bot installation, you can use the `SampleStrategy` installed by the `create-userdir` subcommand (usually `user_data/strategy/sample_strategy.py`). -The bot will search your strategy file within `user_data/strategies` and `freqtrade/strategy`. +The bot will search your strategy file within `user_data/strategies`. +To use other directories, please read the next section about `--strategy-path`. To load a strategy, simply pass the class name (e.g.: `CustomStrategy`) in this parameter. @@ -192,8 +198,8 @@ Backtesting also uses the config specified via `-c/--config`. usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TICKER_INTERVAL] - [--timerange TIMERANGE] [--max_open_trades INT] - [--stake_amount STAKE_AMOUNT] [--fee FLOAT] + [--timerange TIMERANGE] [--max-open-trades INT] + [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--eps] [--dmmp] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export EXPORT] [--export-filename PATH] @@ -205,10 +211,12 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. - --max_open_trades INT - Specify max_open_trades to use. - --stake_amount STAKE_AMOUNT - Specify stake_amount. + --max-open-trades INT + Override the value of the `max_open_trades` + configuration setting. + --stake-amount STAKE_AMOUNT + Override the value of the `stake_amount` configuration + setting. --fee FLOAT Specify fee ratio. Will be applied twice (on trade entry and exit). --eps, --enable-position-stacking @@ -236,12 +244,15 @@ optional arguments: Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -264,17 +275,17 @@ Check the corresponding [Data Downloading](data-download.md) section for more de ## Hyperopt commands To optimize your strategy, you can use hyperopt parameter hyperoptimization -to find optimal parameter values for your stategy. +to find optimal parameter values for your strategy. ``` usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TICKER_INTERVAL] [--timerange TIMERANGE] - [--max_open_trades INT] - [--stake_amount STAKE_AMOUNT] [--fee FLOAT] + [--max-open-trades INT] + [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--hyperopt NAME] [--hyperopt-path PATH] [--eps] [-e INT] - [--spaces {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...]] + [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]] [--dmmp] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] [--continue] [--hyperopt-loss NAME] @@ -286,10 +297,12 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. - --max_open_trades INT - Specify max_open_trades to use. - --stake_amount STAKE_AMOUNT - Specify stake_amount. + --max-open-trades INT + Override the value of the `max_open_trades` + configuration setting. + --stake-amount STAKE_AMOUNT + Override the value of the `stake_amount` configuration + setting. --fee FLOAT Specify fee ratio. Will be applied twice (on trade entry and exit). --hyperopt NAME Specify hyperopt class name which will be used by the @@ -300,9 +313,9 @@ optional arguments: Allow buying the same pair multiple times (position stacking). -e INT, --epochs INT Specify number of epochs (default: 100). - --spaces {all,buy,sell,roi,stoploss} [{all,buy,sell,roi,stoploss} ...] + --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...] Specify which parameters to hyperopt. Space-separated - list. Default: `all`. + list. --dmmp, --disable-max-market-positions Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high @@ -310,7 +323,7 @@ optional arguments: --print-all Print all results, not only the best ones. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. - --print-json Print best result detailization in JSON format. + --print-json Print best results in JSON format. -j JOBS, --job-workers JOBS The number of concurrently running jobs for hyperoptimization (hyperopt worker processes). If -1 @@ -328,18 +341,23 @@ optional arguments: class (IHyperOptLoss). Different functions can generate completely different results, since the target for optimization is different. Built-in - Hyperopt-loss-functions are: DefaultHyperOptLoss, - OnlyProfitHyperOptLoss, SharpeHyperOptLoss (default: - `DefaultHyperOptLoss`). + Hyperopt-loss-functions are: + DefaultHyperOptLoss, OnlyProfitHyperOptLoss, + SharpeHyperOptLoss, SharpeHyperOptLossDaily, + SortinoHyperOptLoss, SortinoHyperOptLossDaily. + (default: `DefaultHyperOptLoss`). Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -350,6 +368,7 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. + ``` ## Edge commands @@ -360,7 +379,7 @@ To know your trade expectancy and winrate against historical data, you can use E usage: freqtrade edge [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TICKER_INTERVAL] [--timerange TIMERANGE] - [--max_open_trades INT] [--stake_amount STAKE_AMOUNT] + [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--stoplosses STOPLOSS_RANGE] optional arguments: @@ -370,10 +389,12 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. - --max_open_trades INT - Specify max_open_trades to use. - --stake_amount STAKE_AMOUNT - Specify stake_amount. + --max-open-trades INT + Override the value of the `max_open_trades` + configuration setting. + --stake-amount STAKE_AMOUNT + Override the value of the `stake_amount` configuration + setting. --fee FLOAT Specify fee ratio. Will be applied twice (on trade entry and exit). --stoplosses STOPLOSS_RANGE @@ -384,12 +405,15 @@ optional arguments: Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -400,6 +424,7 @@ Strategy arguments: Specify strategy class name which will be used by the bot. --strategy-path PATH Specify additional strategy lookup path. + ``` To understand edge and how to read the results, please read the [edge documentation](edge.md). diff --git a/docs/configuration.md b/docs/configuration.md index 73534b6f1..7405fc92e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,78 +34,88 @@ The prevelance for all Options is as follows: - CLI arguments override any other option - Configuration files are used in sequence (last file wins), and override Strategy configurations. -- Strategy configurations are only used if they are not set via configuration or via command line arguments. These options are market with [Strategy Override](#parameters-in-the-strategy) in the below table. +- Strategy configurations are only used if they are not set via configuration or via command line arguments. These options are marked with [Strategy Override](#parameters-in-the-strategy) in the below table. Mandatory parameters are marked as **Required**, which means that they are required to be set in one of the possible ways. -| Command | Description | -|----------|-------------| -| `max_open_trades` | **Required.** Number of trades open your bot will have. If -1 then it is ignored (i.e. potentially unlimited open trades).
***Datatype:*** *Positive integer or -1.* -| `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* -| `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#understand-stake_amount). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Positive float or `"unlimited"`.* -| `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals.
*Defaults to `0.05` (5%).*
***Datatype:*** *Positive Float as ratio.* -| `ticker_interval` | The ticker interval to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *String* -| `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
***Datatype:*** *String* -| `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
***Datatype:*** *Boolean* -| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in the Dry Run mode.
*Defaults to `1000`.*
***Datatype:*** *Float* -| `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `minimal_roi` | **Required.** Set the threshold in percent the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* -| `stoploss` | **Required.** Value of the stoploss in percent used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Float (as ratio)* -| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Boolean* -| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Float* -| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
***Datatype:*** *Float* -| `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled.
***Datatype:*** *Integer* -| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled.
***Datatype:*** *Integer* -| `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#understand-ask_last_balance). -| `bid_strategy.use_order_book` | Enable buying using the rates in Order Book Bids.
***Datatype:*** *Boolean* -| `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in Order Book Bids. *Defaults to `1`.*
***Datatype:*** *Positive Integer* -| `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book.
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The % difference of buy orders and sell orders found in Order Book. A value lesser than 1 means sell orders is greater, while value greater than 1 means buy orders is higher. *Defaults to `0`.*
***Datatype:*** *Float (as ratio)* -| `ask_strategy.use_order_book` | Enable selling of open trades using Order Book Asks.
***Datatype:*** *Boolean* -| `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
***Datatype:*** *Positive Integer* -| `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
***Datatype:*** *Positive Integer* -| `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
***Datatype:*** *Boolean* -| `ask_strategy.sell_profit_only` | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `ask_strategy.ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
***Datatype:*** *Boolean* -| `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* -| `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
***Datatype:*** *Dict* -| `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
***Datatype:*** *String* -| `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
***Datatype:*** *Boolean* -| `exchange.key` | API key to use for the exchange. Only required when you are in production mode. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#dynamic-pairlists)).
***Datatype:*** *List* -| `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)).
***Datatype:*** *List* -| `exchange.ccxt_config` | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
***Datatype:*** *Dict* -| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
***Datatype:*** *Dict* -| `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
***Datatype:*** *Positive Integer* +| Parameter | Description | +|------------|-------------| +| `max_open_trades` | **Required.** Number of open trades your bot is allowed to have. Only one open trade per pair is possible, so the length of your pairlist is another limitation which can apply. If -1 then it is ignored (i.e. potentially unlimited open trades, limited by the pairlist). [More information below](#configuring-amount-per-trade).
**Datatype:** Positive integer or -1. +| `stake_currency` | **Required.** Crypto-currency used for trading. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String +| `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Positive float or `"unlimited"`. +| `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade).
*Defaults to `0.99` 99%).*
**Datatype:** Positive float between `0.1` and `1.0`. +| `amend_last_stake_amount` | Use reduced last stake amount if necessary. [More information below](#configuring-amount-per-trade).
*Defaults to `false`.*
**Datatype:** Boolean +| `last_stake_amount_min_ratio` | Defines minimum stake amount that has to be left and executed. Applies only to the last stake amount when it's amended to a reduced value (i.e. if `amend_last_stake_amount` is set to `true`). [More information below](#configuring-amount-per-trade).
*Defaults to `0.5`.*
**Datatype:** Float (as ratio) +| `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals.
*Defaults to `0.05` (5%).*
**Datatype:** Positive Float as ratio. +| `ticker_interval` | The timeframe (ticker interval) to use (e.g `1m`, `5m`, `15m`, `30m`, `1h` ...). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** String +| `fiat_display_currency` | Fiat currency used to show your profits. [More information below](#what-values-can-be-used-for-fiat_display_currency).
**Datatype:** String +| `dry_run` | **Required.** Define if the bot must be in Dry Run or production mode.
*Defaults to `true`.*
**Datatype:** Boolean +| `dry_run_wallet` | Define the starting amount in stake currency for the simulated wallet used by the bot running in the Dry Run mode.
*Defaults to `1000`.*
**Datatype:** Float +| `cancel_open_orders_on_exit` | Cancel open orders when the `/stop` RPC command is issued, `Ctrl+C` is pressed or the bot dies unexpectedly. When set to `true`, this allows you to use `/stop` to cancel unfilled and partially filled orders in the event of a market crash. It does not impact open positions.
*Defaults to `false`.*
**Datatype:** Boolean +| `process_only_new_candles` | Enable processing of indicators only when new candles arrive. If false each loop populates the indicators, this will mean the same candle is processed many times creating system load but can be useful of your strategy depends on tick data not only candle. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `minimal_roi` | **Required.** Set the threshold in percent the bot will use to sell a trade. [More information below](#understand-minimal_roi). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict +| `stoploss` | **Required.** Value of the stoploss in percent used by the bot. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Float (as ratio) +| `trailing_stop` | Enables trailing stoploss (based on `stoploss` in either configuration or strategy file). More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Boolean +| `trailing_stop_positive` | Changes stoploss once profit has been reached. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Float +| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `0.0` (no offset).*
**Datatype:** Float +| `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer +| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled. [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Integer +| `bid_strategy.price_side` | Select the side of the spread the bot should look at to get the buy rate. [More information below](#buy-price-side).
*Defaults to `bid`.*
**Datatype:** String (either `ask` or `bid`). +| `bid_strategy.ask_last_balance` | **Required.** Set the bidding price. More information [below](#buy-price-without-orderbook-enabled). +| `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled).
**Datatype:** Boolean +| `bid_strategy.order_book_top` | Bot will use the top N rate in Order Book Bids to buy. I.e. a value of 2 will allow the bot to pick the 2nd bid rate in [Order Book Bids](#buy-price-with-orderbook-enabled).
*Defaults to `1`.*
**Datatype:** Positive Integer +| `bid_strategy. check_depth_of_market.enabled` | Do not buy if the difference of buy orders and sell orders is met in Order Book. [Check market depth](#check-depth-of-market).
*Defaults to `false`.*
**Datatype:** Boolean +| `bid_strategy. check_depth_of_market.bids_to_ask_delta` | The difference ratio of buy orders and sell orders found in Order Book. A value below 1 means sell order size is greater, while value greater than 1 means buy order size is higher. [Check market depth](#check-depth-of-market)
*Defaults to `0`.*
**Datatype:** Float (as ratio) +| `ask_strategy.price_side` | Select the side of the spread the bot should look at to get the sell rate. [More information below](#sell-price-side).
*Defaults to `ask`.*
**Datatype:** String (either `ask` or `bid`). +| `ask_strategy.use_order_book` | Enable selling of open trades using [Order Book Asks](#sell-price-with-orderbook-enabled).
**Datatype:** Boolean +| `ask_strategy.order_book_min` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer +| `ask_strategy.order_book_max` | Bot will scan from the top min to max Order Book Asks searching for a profitable rate.
*Defaults to `1`.*
**Datatype:** Positive Integer +| `ask_strategy.use_sell_signal` | Use sell signals produced by the strategy in addition to the `minimal_roi`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `true`.*
**Datatype:** Boolean +| `ask_strategy.sell_profit_only` | Wait until the bot makes a positive profit before taking a sell decision. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `ask_strategy.ignore_roi_if_buy_signal` | Do not sell if the buy signal is still active. This setting takes preference over `minimal_roi` and `use_sell_signal`. [Strategy Override](#parameters-in-the-strategy).
*Defaults to `false`.*
**Datatype:** Boolean +| `order_types` | Configure order-types depending on the action (`"buy"`, `"sell"`, `"stoploss"`, `"stoploss_on_exchange"`). [More information below](#understand-order_types). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict +| `order_time_in_force` | Configure time in force for buy and sell orders. [More information below](#understand-order_time_in_force). [Strategy Override](#parameters-in-the-strategy).
**Datatype:** Dict +| `exchange.name` | **Required.** Name of the exchange class to use. [List below](#user-content-what-values-for-exchangename).
**Datatype:** String +| `exchange.sandbox` | Use the 'sandbox' version of the exchange, where the exchange provides a sandbox for risk-free integration. See [here](sandbox-testing.md) in more details.
**Datatype:** Boolean +| `exchange.key` | API key to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `exchange.secret` | API secret to use for the exchange. Only required when you are in production mode.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `exchange.password` | API password to use for the exchange. Only required when you are in production mode and for exchanges that use password for API requests.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `exchange.pair_whitelist` | List of pairs to use by the bot for trading and to check for potential trades during backtesting. Not used by VolumePairList (see [below](#dynamic-pairlists)).
**Datatype:** List +| `exchange.pair_blacklist` | List of pairs the bot must absolutely avoid for trading and backtesting (see [below](#dynamic-pairlists)).
**Datatype:** List +| `exchange.ccxt_config` | Additional CCXT parameters passed to the regular ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict +| `exchange.ccxt_async_config` | Additional CCXT parameters passed to the async ccxt instance. Parameters may differ from exchange to exchange and are documented in the [ccxt documentation](https://ccxt.readthedocs.io/en/latest/manual.html#instantiation)
**Datatype:** Dict +| `exchange.markets_refresh_interval` | The interval in minutes in which markets are reloaded.
*Defaults to `60` minutes.*
**Datatype:** Positive Integer | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. -| `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
***Datatype:*** *Boolean* -| `pairlists` | Define one or more pairlists to be used. [More information below](#dynamic-pairlists).
*Defaults to `StaticPairList`.*
***Datatype:*** *List of Dicts* -| `telegram.enabled` | Enable the usage of Telegram.
***Datatype:*** *Boolean* -| `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `webhook.enabled` | Enable usage of Webhook notifications
***Datatype:*** *Boolean* -| `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
***Datatype:*** *String* -| `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* -| `webhook.webhooksell` | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* -| `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentationV](webhook-config.md) for more details.
***Datatype:*** *String* -| `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Boolean* -| `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *IPv4* -| `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
***Datatype:*** *Integer between 1024 and 65535* -| `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details. **Keep it in secret, do not disclose publicly.**
***Datatype:*** *String* -| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite://` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
***Datatype:*** *String, SQLAlchemy connect string* -| `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
***Datatype:*** *Enum, either `stopped` or `running`* -| `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
***Datatype:*** *Boolean* -| `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
***Datatype:*** *ClassName* -| `strategy_path` | Adds an additional strategy lookup path (must be a directory).
***Datatype:*** *String* -| `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
***Datatype:*** *Positive Integer* -| `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages.
*Defaults to `60` seconds.*
***Datatype:*** *Positive Integer or 0* -| `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
***Datatype:*** *Boolean* -| `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
***Datatype:*** *String* -| `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
***Datatype:*** *String* +| `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean +| `pairlists` | Define one or more pairlists to be used. [More information below](#dynamic-pairlists).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts +| `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean +| `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `webhook.enabled` | Enable usage of Webhook notifications
**Datatype:** Boolean +| `webhook.url` | URL for the webhook. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `webhook.webhookbuy` | Payload to send on buy. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `webhook.webhookbuycancel` | Payload to send on buy order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `webhook.webhooksell` | Payload to send on sell. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `webhook.webhooksellcancel` | Payload to send on sell order cancel. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `webhook.webhookstatus` | Payload to send on status calls. Only required if `webhook.enabled` is `true`. See the [webhook documentation](webhook-config.md) for more details.
**Datatype:** String +| `api_server.enabled` | Enable usage of API Server. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** Boolean +| `api_server.listen_ip_address` | Bind IP address. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** IPv4 +| `api_server.listen_port` | Bind Port. See the [API Server documentation](rest-api.md) for more details.
**Datatype:** Integer between 1024 and 65535 +| `api_server.username` | Username for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `api_server.password` | Password for API server. See the [API Server documentation](rest-api.md) for more details.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String +| `db_url` | Declares database URL to use. NOTE: This defaults to `sqlite:///tradesv3.dryrun.sqlite` if `dry_run` is `true`, and to `sqlite:///tradesv3.sqlite` for production instances.
**Datatype:** String, SQLAlchemy connect string +| `initial_state` | Defines the initial application state. More information below.
*Defaults to `stopped`.*
**Datatype:** Enum, either `stopped` or `running` +| `forcebuy_enable` | Enables the RPC Commands to force a buy. More information below.
**Datatype:** Boolean +| `strategy` | **Required** Defines Strategy class to use. Recommended to be set via `--strategy NAME`.
**Datatype:** ClassName +| `strategy_path` | Adds an additional strategy lookup path (must be a directory).
**Datatype:** String +| `internals.process_throttle_secs` | Set the process throttle. Value in second.
*Defaults to `5` seconds.*
**Datatype:** Positive Integer +| `internals.heartbeat_interval` | Print heartbeat message every N seconds. Set to 0 to disable heartbeat messages.
*Defaults to `60` seconds.*
**Datatype:** Positive Integer or 0 +| `internals.sd_notify` | Enables use of the sd_notify protocol to tell systemd service manager about changes in the bot state and issue keep-alive pings. See [here](installation.md#7-optional-configure-freqtrade-as-a-systemd-service) for more details.
**Datatype:** Boolean +| `logfile` | Specifies logfile name. Uses a rolling strategy for log file rotation for 10 files with the 1MB limit per file.
**Datatype:** String +| `user_data_dir` | Directory containing user data.
*Defaults to `./user_data/`*.
**Datatype:** String +| `dataformat_ohlcv` | Data format to use to store historical candle (OHLCV) data.
*Defaults to `json`*.
**Datatype:** String +| `dataformat_trades` | Data format to use to store historical trades data.
*Defaults to `jsongz`*.
**Datatype:** String ### Parameters in the strategy @@ -124,24 +134,63 @@ Values set in the configuration file always overwrite values set in the strategy * `order_time_in_force` * `stake_currency` * `stake_amount` +* `unfilledtimeout` * `use_sell_signal` (ask_strategy) * `sell_profit_only` (ask_strategy) * `ignore_roi_if_buy_signal` (ask_strategy) -### Understand stake_amount +### Configuring amount per trade -The `stake_amount` configuration parameter is an amount of crypto-currency your bot will use for each trade. +There are several methods to configure how much of the stake currency the bot will use to enter a trade. All methods respect the [available balance configuration](#available-balance) as explained below. -The minimal configuration value is 0.0001. Please check your exchange's trading minimums to avoid problems. +#### Available balance + +By default, the bot assumes that the `complete amount - 1%` is at it's disposal, and when using [dynamic stake amount](#dynamic-stake-amount), it will split the complete balance into `max_open_trades` buckets per trade. +Freqtrade will reserve 1% for eventual fees when entering a trade and will therefore not touch that by default. + +You can configure the "untouched" amount by using the `tradable_balance_ratio` setting. + +For example, if you have 10 ETH available in your wallet on the exchange and `tradable_balance_ratio=0.5` (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers this as available balance. The rest of the wallet is untouched by the trades. + +!!! Warning + The `tradable_balance_ratio` setting applies to the current balance (free balance + tied up in trades). Therefore, assuming the starting balance of 1000, a configuration with `tradable_balance_ratio=0.99` will not guarantee that 10 currency units will always remain available on the exchange. For example, the free amount may reduce to 5 units if the total balance is reduced to 500 (either by a losing streak, or by withdrawing balance). + +#### Amend last stake amount + +Assuming we have the tradable balance of 1000 USDT, `stake_amount=400`, and `max_open_trades=3`. +The bot would open 2 trades, and will be unable to fill the last trading slot, since the requested 400 USDT are no longer available, since 800 USDT are already tied in other trades. + +To overcome this, the option `amend_last_stake_amount` can be set to `True`, which will enable the bot to reduce stake_amount to the available balance in order to fill the last trade slot. + +In the example above this would mean: + +- Trade1: 400 USDT +- Trade2: 400 USDT +- Trade3: 200 USDT + +!!! Note + This option only applies with [Static stake amount](#static-stake-amount) - since [Dynamic stake amount](#dynamic-stake-amount) divides the balances evenly. + +!!! Note + The minimum last stake amount can be configured using `amend_last_stake_amount` - which defaults to 0.5 (50%). This means that the minimum stake amount that's ever used is `stake_amount * 0.5`. This avoids very low stake amounts, that are close to the minimum tradable amount for the pair and can be refused by the exchange. + +#### Static stake amount + +The `stake_amount` configuration statically configures the amount of stake-currency your bot will use for each trade. + +The minimal configuration value is 0.0001, however, please check your exchange's trading minimums for the stake currency you're using to avoid problems. This setting works in combination with `max_open_trades`. The maximum capital engaged in trades is `stake_amount * max_open_trades`. For example, the bot will at most use (0.05 BTC x 3) = 0.15 BTC, assuming a configuration of `max_open_trades=3` and `stake_amount=0.05`. -To allow the bot to trade all the available `stake_currency` in your account set +!!! Note + This setting respects the [available balance configuration](#available-balance). -```json -"stake_amount" : "unlimited", -``` +#### Dynamic stake amount + +Alternatively, you can use a dynamic stake amount, which will use the available balance on the exchange, and divide that equally by the amount of allowed trades (`max_open_trades`). + +To configure this, set `stake_amount="unlimited"`. We also recommend to set `tradable_balance_ratio=0.99` (99%) - to keep a minimum balance for eventual fees. In this case a trade amount is calculated as: @@ -149,6 +198,16 @@ In this case a trade amount is calculated as: currency_balance / (max_open_trades - current_open_trades) ``` +To allow the bot to trade all the available `stake_currency` in your account (minus `tradable_balance_ratio`) set + +```json +"stake_amount" : "unlimited", +"tradable_balance_ratio": 0.99, +``` + +!!! Note + This configuration will allow increasing / decreasing stakes depending on the performance of the bot (lower stake if bot is loosing, higher stakes if the bot has a winning record, since higher balances are available). + !!! Note "When using Dry-Run Mode" When using `"stake_amount" : "unlimited",` in combination with Dry-Run, the balance will be simulated starting with a stake of `dry_run_wallet` which will evolve over time. It is therefore important to set `dry_run_wallet` to a sensible value (like 0.05 or 0.01 for BTC and 1000 or 100 for USDT, for example), otherwise it may simulate trades with 100 BTC (or more) or 0.05 USDT (or less) at once - which may not correspond to your real available balance or is less than the exchange minimal limit for the order amount for the stake currency. @@ -207,13 +266,6 @@ before asking the strategy if we should buy or a sell an asset. After each wait every opened trade wether or not we should sell, and for all the remaining pairs (either the dynamic list of pairs or the static list of pairs) if we should buy. -### Understand ask_last_balance - -The `ask_last_balance` configuration parameter sets the bidding price. Value `0.0` will use `ask` price, `1.0` will -use the `last` price and values between those interpolate between ask and last -price. Using `ask` price will guarantee quick success in bid, but bot will also -end up paying more then would probably have been necessary. - ### Understand order_types The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. @@ -233,7 +285,7 @@ If this is configured, the following 4 values (`buy`, `sell`, `stoploss` and The below is the default which is used if this is not configured in either strategy or configuration file. Since `stoploss_on_exchange` uses limit orders, the exchange needs 2 prices, the stoploss_price and the Limit price. -`stoploss` defines the stop-price - and limit should be slightly below this. This defaults to 0.99 / 1%. +`stoploss` defines the stop-price - and limit should be slightly below this. This defaults to 0.99 / 1% (configurable via `stoploss_on_exchange_limit_ratio`). Calculation example: we bought the asset at 100$. Stop-price is 95$, then limit would be `95 * 0.99 = 94.05$` - so the stoploss will happen between 95$ and 94.05$. @@ -291,7 +343,7 @@ This is most of the time the default time in force. It means the order will rema on exchange till it is canceled by user. It can be fully or partially fulfilled. If partially fulfilled, the remaining will stay on the exchange till cancelled. -**FOK (Full Or Kill):** +**FOK (Fill Or Kill):** It means if the order is not executed immediately AND fully then it is canceled by the exchange. @@ -321,16 +373,18 @@ The possible values are: `gtc` (default), `fok` or `ioc`. Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency exchange markets and trading APIs. The complete up-to-date list can be found in the -[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). However, the bot was tested -with only Bittrex and Binance. - -The bot was tested with the following exchanges: +[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). + However, the bot was tested by the development team with only Bittrex, Binance and Kraken, + so the these are the only officially supported exhanges: - [Bittrex](https://bittrex.com/): "bittrex" - [Binance](https://www.binance.com/): "binance" +- [Kraken](https://kraken.com/): "kraken" Feel free to test other exchanges and submit your PR to improve the bot. +Some exchanges require special configuration, which can be found on the [Exchange-specific Notes](exchanges.md) documentation page. + #### Sample exchange configuration A exchange configuration for "binance" would look as follows: @@ -360,7 +414,7 @@ Advanced options can be configured using the `_ft_has_params` setting, which wil Available options are listed in the exchange-class as `_ft_has_default`. -For example, to test the order type `FOK` with Kraken, and modify candle_limit to 200 (so you only get 200 candles per call): +For example, to test the order type `FOK` with Kraken, and modify candle limit to 200 (so you only get 200 candles per API call): ```json "exchange": { @@ -393,6 +447,108 @@ The valid values are: "BTC", "ETH", "XRP", "LTC", "BCH", "USDT" ``` +## Prices used for orders + +Prices for regular orders can be controlled via the parameter structures `bid_strategy` for buying and `ask_strategy` for selling. +Prices are always retrieved right before an order is placed, either by querying the exchange tickers or by using the orderbook data. + +!!! Note + Orderbook data used by Freqtrade are the data retrieved from exchange by the ccxt's function `fetch_order_book()`, i.e. are usually data from the L2-aggregated orderbook, while the ticker data are the structures returned by the ccxt's `fetch_ticker()`/`fetch_tickers()` functions. Refer to the ccxt library [documentation](https://github.com/ccxt/ccxt/wiki/Manual#market-data) for more details. + +### Buy price + +#### Check depth of market + +When check depth of market is enabled (`bid_strategy.check_depth_of_market.enabled=True`), the buy signals are filtered based on the orderbook depth (sum of all amounts) for each orderbook side. + +Orderbook `bid` (buy) side depth is then divided by the orderbook `ask` (sell) side depth and the resulting delta is compared to the value of the `bid_strategy.check_depth_of_market.bids_to_ask_delta` parameter. The buy order is only executed if the orderbook delta is greater than or equal to the configured delta value. + +!!! Note + A delta value below 1 means that `ask` (sell) orderbook side depth is greater than the depth of the `bid` (buy) orderbook side, while a value greater than 1 means opposite (depth of the buy side is higher than the depth of the sell side). + +#### Buy price side + +The configuration setting `bid_strategy.price_side` defines the side of the spread the bot looks for when buying. + +The following displays an orderbook. + +``` explanation +... +103 +102 +101 # ask +-------------Current spread +99 # bid +98 +97 +... +``` + +If `bid_strategy.price_side` is set to `"bid"`, then the bot will use 99 as buying price. +In line with that, if `bid_strategy.price_side` is set to `"ask"`, then the bot will use 101 as buying price. + +Using `ask` price often guarantees quicker filled orders, but the bot can also end up paying more than what would have been necessary. +Taker fees instead of maker fees will most likely apply even when using limit buy orders. +Also, prices at the "ask" side of the spread are higher than prices at the "bid" side in the orderbook, so the order behaves similar to a market order (however with a maximum price). + +#### Buy price with Orderbook enabled + +When buying with the orderbook enabled (`bid_strategy.use_order_book=True`), Freqtrade fetches the `bid_strategy.order_book_top` entries from the orderbook and then uses the entry specified as `bid_strategy.order_book_top` on the configured side (`bid_strategy.price_side`) of the orderbook. 1 specifies the topmost entry in the orderbook, while 2 would use the 2nd entry in the orderbook, and so on. + +#### Buy price without Orderbook enabled + +The following section uses `side` as the configured `bid_strategy.price_side`. + +When not using orderbook (`bid_strategy.use_order_book=False`), Freqtrade uses the best `side` price from the ticker if it's below the `last` traded price from the ticker. Otherwise (when the `side` price is above the `last` price), it calculates a rate between `side` and `last` price. + +The `bid_strategy.ask_last_balance` configuration parameter controls this. A value of `0.0` will use `side` price, while `1.0` will use the `last` price and values between those interpolate between ask and last price. + +### Sell price + +#### Sell price side + +The configuration setting `ask_strategy.price_side` defines the side of the spread the bot looks for when selling. + +The following displays an orderbook: + +``` explanation +... +103 +102 +101 # ask +-------------Current spread +99 # bid +98 +97 +... +``` + +If `ask_strategy.price_side` is set to `"ask"`, then the bot will use 101 as selling price. +In line with that, if `ask_strategy.price_side` is set to `"bid"`, then the bot will use 99 as selling price. + +#### Sell price with Orderbook enabled + +When selling with the orderbook enabled (`ask_strategy.use_order_book=True`), Freqtrade fetches the `ask_strategy.order_book_max` entries in the orderbook. Then each of the orderbook steps between `ask_strategy.order_book_min` and `ask_strategy.order_book_max` on the configured orderbook side are validated for a profitable sell-possibility based on the strategy configuration (`minimal_roi` conditions) and the sell order is placed at the first profitable spot. + +!!! Note + Using `order_book_max` higher than `order_book_min` only makes sense when ask_strategy.price_side is set to `"ask"`. + +The idea here is to place the sell order early, to be ahead in the queue. + +A fixed slot (mirroring `bid_strategy.order_book_top`) can be defined by setting `ask_strategy.order_book_min` and `ask_strategy.order_book_max` to the same number. + +!!! Warning "Order_book_max > 1 - increased risks for stoplosses!" + Using `ask_strategy.order_book_max` higher than 1 will increase the risk the stoploss on exchange is cancelled too early, since an eventual [stoploss on exchange](#understand-order_types) will be cancelled as soon as the order is placed. + Also, the sell order will remain on the exchange for `unfilledtimeout.sell` (or until it's filled) - which can lead to missed stoplosses (with or without using stoploss on exchange). + +!!! Warning "Order_book_max > 1 in dry-run" + Using `ask_strategy.order_book_max` higher than 1 will result in improper dry-run results (significantly better than real orders executed on exchange), since dry-run assumes orders to be filled almost instantly. + It is therefore advised to not use this setting for dry-runs. + +#### Sell price without Orderbook enabled + +When not using orderbook (`ask_strategy.use_order_book=False`), the price at the `ask_strategy.price_side` side (defaults to `"ask"`) from the ticker will be used as the sell price. + ## Pairlists Pairlists define the list of pairs that the bot should trade. @@ -410,6 +566,7 @@ Inactive markets and blacklisted pairs are always removed from the resulting `pa * [`VolumePairList`](#volume-pair-list) * [`PrecisionFilter`](#precision-filter) * [`PriceFilter`](#price-pair-filter) +* [`SpreadFilter`](#spread-filter) !!! Tip "Testing pairlists" Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) subcommand to test your configuration quickly. @@ -428,12 +585,16 @@ It uses configuration from `exchange.pair_whitelist` and `exchange.pair_blacklis #### Volume Pair List -`VolumePairList` selects `number_assets` top pairs based on `sort_key`, which can be one of `askVolume`, `bidVolume` and `quoteVolume` and defaults to `quoteVolume`. +`VolumePairList` selects `number_assets` top pairs based on `sort_key`, which can only be `quoteVolume`. `VolumePairList` considers outputs of previous pairlists unless it's the first configured pairlist, it does not consider `pair_whitelist`, but selects the top assets from all available markets (with matching stake-currency) on the exchange. `refresh_period` allows setting the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes). +`VolumePairList` is based on the ticker data, as reported by the ccxt library: + +* The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours. + ```json "pairlists": [{ "method": "VolumePairList", @@ -458,6 +619,12 @@ Min price precision is 8 decimals. If price is 0.00000011 - one step would be 0. These pairs are dangerous since it may be impossible to place the desired stoploss - and often result in high losses. +#### Spread Filter + +Removes pairs that have a difference between asks and bids above the specified ratio (default `0.005`). +Example: +If `DOGE/BTC` maximum bid is 0.00000026 and minimum ask is 0.00000027 the ratio is calculated as: `1 - bid/ask ~= 0.037` which is `> 0.005` + ### Full Pairlist example The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting by `quoteVolume` and applies both [`PrecisionFilter`](#precision-filter) and [`PriceFilter`](#price-pair-filter), filtering all assets where 1 priceunit is > 1%. @@ -509,12 +676,25 @@ Once you will be happy with your bot performance running in the Dry-run mode, yo !!! Note A simulated wallet is available during dry-run mode, and will assume a starting capital of `dry_run_wallet` (defaults to 1000). +### Considerations for dry-run + +* API-keys may or may not be provided. Only Read-Only operations (i.e. operations that do not alter account state) on the exchange are performed in the dry-run mode. +* Wallets (`/balance`) are simulated. +* Orders are simulated, and will not be posted to the exchange. +* In combination with `stoploss_on_exchange`, the stop_loss price is assumed to be filled. +* Open orders (not trades, which are stored in the database) are reset on bot restart. + ## Switch to production mode In production mode, the bot will engage your money. Be careful, since a wrong strategy can lose all your money. Be aware of what you are doing when you run it in production mode. +### Setup your exchange account + +You will need to create API Keys (usually you get `key` and `secret`, some exchanges require an additional `password`) from the Exchange website and you'll need to insert this into the appropriate fields in the configuration or when asked by the `freqtrade new-config` command. +API Keys are usually only required for live trading (trading for real money, bot running in "production mode", executing real orders on the exchange) and are not required for the bot running in dry-run (trade simulation) mode. When you setup the bot in dry-run mode, you may fill these fields with empty values. + ### To switch your bot in production mode **Edit your `config.json` file.** @@ -536,9 +716,6 @@ you run it in production mode. } ``` -!!! Note - If you have an exchange API key yet, [see our tutorial](installation.md#setup-your-exchange-account). - You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange. ### Using proxy with Freqtrade @@ -563,7 +740,7 @@ freqtrade ## Embedding Strategies -FreqTrade provides you with with an easy way to embed the strategy into your configuration file. +Freqtrade provides you with with an easy way to embed the strategy into your configuration file. This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. diff --git a/docs/data-analysis.md b/docs/data-analysis.md index 115ce1916..fc4693b17 100644 --- a/docs/data-analysis.md +++ b/docs/data-analysis.md @@ -8,6 +8,27 @@ You can analyze the results of backtests and trading history easily using Jupyte * Don't forget to start a Jupyter notebook server from within your conda or venv environment or use [nb_conda_kernels](https://github.com/Anaconda-Platform/nb_conda_kernels)* * Copy the example notebook before use so your changes don't get clobbered with the next freqtrade update. +### Using virtual environment with system-wide Jupyter installation + +Sometimes it can be desired to use a system-wide installation of Jupyter notebook, and use a jupyter kernel from the virtual environment. +This prevents you from installing the full jupyter suite multiple times per system, and provides an easy way to switch between tasks (freqtrade / other analytics tasks). + +For this to work, first activate your virtual environment and run the following commands: + +``` bash +# Activate virtual environment +source .env/bin/activate + +pip install ipykernel +ipython kernel install --user --name=freqtrade +# Restart jupyter (lab / notebook) +# select kernel "freqtrade" in the notebook +``` + +!!! Note + This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get jupyter notebooks up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community). + + ## Fine print Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually. diff --git a/docs/data-download.md b/docs/data-download.md index 1f03b124a..903d62854 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -12,6 +12,152 @@ Otherwise `--exchange` becomes mandatory. If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. Be carefull though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded. +### Usage + +``` +usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-p PAIRS [PAIRS ...]] + [--pairs-file FILE] [--days INT] [--dl-trades] [--exchange EXCHANGE] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] + [--erase] [--data-format-ohlcv {json,jsongz}] [--data-format-trades {json,jsongz}] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space-separated. + --pairs-file FILE File containing a list of pairs to download. + --days INT Download data for given number of days. + --dl-trades Download trades instead of OHLCV data. The bot will resample trades to the desired timeframe as specified as + --timeframes/-t. + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] + Specify which tickers to download. Space-separated list. Default: `1m 5m`. + --erase Clean all existing data for the selected exchange/pairs/timeframes. + --data-format-ohlcv {json,jsongz} + Storage format for downloaded candle (OHLCV) data. (default: `json`). + --data-format-trades {json,jsongz} + Storage format for downloaded trades data. (default: `jsongz`). + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` + +### Data format + +Freqtrade currently supports 2 dataformats, `json` (plain "text" json files) and `jsongz` (a gzipped version of json files). +By default, OHLCV data is stored as `json` data, while trades data is stored as `jsongz` data. + +This can be changed via the `--data-format-ohlcv` and `--data-format-trades` parameters respectivly. + +If the default dataformat has been changed during download, then the keys `dataformat_ohlcv` and `dataformat_trades` in the configuration file need to be adjusted to the selected dataformat as well. + +!!! Note + You can convert between data-formats using the [convert-data](#subcommand-convert-data) and [convert-trade-data](#subcommand-convert-trade-data) methods. + +#### Subcommand convert data + +``` +usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] --format-from + {json,jsongz} --format-to {json,jsongz} + [--erase] + [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. + --format-from {json,jsongz} + Source format for data conversion. + --format-to {json,jsongz} + Destination format for data conversion. + --erase Clean all existing data for the selected + exchange/pairs/timeframes. + -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...] + Specify which tickers to download. Space-separated + list. Default: `1m 5m`. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` + +##### Example converting data + +The following command will convert all candle (OHLCV) data available in `~/.freqtrade/data/binance` from json to jsongz, saving diskspace in the process. +It'll also remove original json data files (`--erase` parameter). + +``` bash +freqtrade convert-data --format-from json --format-to jsongz --data-dir ~/.freqtrade/data/binance -t 5m 15m --erase +``` + +#### Subcommand convert-trade data + +``` +usage: freqtrade convert-trade-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [-p PAIRS [PAIRS ...]] --format-from + {json,jsongz} --format-to {json,jsongz} + [--erase] + +optional arguments: + -h, --help show this help message and exit + -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] + Show profits for only these pairs. Pairs are space- + separated. + --format-from {json,jsongz} + Source format for data conversion. + --format-to {json,jsongz} + Destination format for data conversion. + --erase Clean all existing data for the selected + exchange/pairs/timeframes. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` + +##### Example converting trades + +The following command will convert all available trade-data in `~/.freqtrade/data/kraken` from jsongz to json. +It'll also remove original jsongz data files (`--erase` parameter). + +``` bash +freqtrade convert-trade-data --format-from jsongz --format-to json --data-dir ~/.freqtrade/data/kraken --erase +``` + ### Pairs file In alternative to the whitelist from `config.json`, a `pairs.json` file can be used. @@ -46,15 +192,15 @@ Then run: freqtrade download-data --exchange binance ``` -This will download ticker data for all the currency pairs you defined in `pairs.json`. +This will download historical candle (OHLCV) data for all the currency pairs you defined in `pairs.json`. ### Other Notes - To use a different directory than the exchange specific default, use `--datadir user_data/data/some_directory`. -- To change the exchange used to download the tickers, please use a different configuration file (you'll probably need to adjust ratelimits etc.) +- To change the exchange used to download the historical data from, please use a different configuration file (you'll probably need to adjust ratelimits etc.) - To use `pairs.json` from some other directory, use `--pairs-file some_other_dir/pairs.json`. -- To download ticker data for only 10 days, use `--days 10` (defaults to 30 days). -- Use `--timeframes` to specify which tickers to download. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute tickers. +- To download historical candle (OHLCV) data for only 10 days, use `--days 10` (defaults to 30 days). +- Use `--timeframes` to specify what timeframe download the historical candle (OHLCV) data for. Default is `--timeframes 1m 5m` which will download 1-minute and 5-minute data. - To use exchange, timeframe and list of pairs as defined in your configuration file, use the `-c/--config` option. With this, the script uses the whitelist defined in the config as the list of currency pairs to download data for and does not require the pairs.json file. You can combine `-c/--config` with most other options. ### Trades (tick) data diff --git a/docs/deprecated.md b/docs/deprecated.md index 349d41a09..a7b57b10e 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -24,3 +24,13 @@ and in freqtrade 2019.7 (master branch). `--live` in the context of backtesting allowed to download the latest tick data for backtesting. Did only download the latest 500 candles, so was ineffective in getting good backtest data. Removed in 2019-7-dev (develop branch) and in freqtrade 2019-8 (master branch) + +### Allow running multiple pairlists in sequence + +The former `"pairlist"` section in the configuration has been removed, and is replaced by `"pairlists"` - being a list to specify a sequence of pairlists. + +The old section of configuration parameters (`"pairlist"`) has been deprecated in 2019.11 and has been removed in 2020.4. + +### deprecation of bidVolume and askVolume from volumepairlist + +Since only quoteVolume can be compared between assets, the other options (bidVolume, askVolume) have been deprecated in 2020.4. diff --git a/docs/developer.md b/docs/developer.md index 5b07aff03..34b2f1ba5 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -1,6 +1,6 @@ # Development Help -This page is intended for developers of FreqTrade, people who want to contribute to the FreqTrade codebase or documentation, or people who want to understand the source code of the application they're running. +This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel in [slack](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) where you can ask questions. @@ -153,7 +153,7 @@ In VolumePairList, this implements different methods of sorting, does early vali ## Implement a new Exchange (WIP) !!! Note - This section is a Work in Progress and is not a complete guide on how to test a new exchange with FreqTrade. + This section is a Work in Progress and is not a complete guide on how to test a new exchange with Freqtrade. Most exchanges supported by CCXT should work out of the box. @@ -165,7 +165,7 @@ Since CCXT does not provide unification for Stoploss On Exchange yet, we'll need ### Incomplete candles -While fetching OHLCV data, we're may end up getting incomplete candles (Depending on the exchange). +While fetching candle (OHLCV) data, we may end up getting incomplete candles (depending on the exchange). To demonstrate this, we'll use daily candles (`"1d"`) to keep things simple. We query the api (`ct.fetch_ohlcv()`) for the timeframe and look at the date of the last entry. If this entry changes or shows the date of a "incomplete" candle, then we should drop this since having incomplete candles is problematic because indicators assume that only complete candles are passed to them, and will generate a lot of false buy signals. By default, we're therefore removing the last candle assuming it's incomplete. @@ -174,14 +174,14 @@ To check how the new exchange behaves, you can use the following snippet: ``` python import ccxt from datetime import datetime -from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.data.converter import ohlcv_to_dataframe ct = ccxt.binance() timeframe = "1d" pair = "XLM/BTC" # Make sure to use a pair that exists on that exchange! raw = ct.fetch_ohlcv(pair, timeframe=timeframe) # convert to dataframe -df1 = parse_ticker_dataframe(raw, timeframe, pair=pair, drop_incomplete=False) +df1 = ohlcv_to_dataframe(raw, timeframe, pair=pair, drop_incomplete=False) print(df1.tail(1)) print(datetime.utcnow()) @@ -234,7 +234,7 @@ git checkout -b new_release Determine if crucial bugfixes have been made between this commit and the current state, and eventually cherry-pick these. -* Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7-1` should we need to do a second release that month. +* Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi. * Commit this part * push that branch to the remote and create a PR against the master branch @@ -266,4 +266,24 @@ Once the PR against master is merged (best right after merging): * Use the button "Draft a new release" in the Github UI (subsection releases). * Use the version-number specified as tag. * Use "master" as reference (this step comes after the above PR is merged). -* Use the above changelog as release comment (as codeblock). +* Use the above changelog as release comment (as codeblock) + +## Releases + +### pypi + +To create a pypi release, please run the following commands: + +Additional requirement: `wheel`, `twine` (for uploading), account on pypi with proper permissions. + +``` bash +python setup.py sdist bdist_wheel + +# For pypi test (to check if some change to the installation did work) +twine upload --repository-url https://test.pypi.org/legacy/ dist/* + +# For production: +twine upload dist/* +``` + +Please don't push non-releases to the productive / real pypi instance. diff --git a/docs/docker.md b/docs/docker.md index ff5bf7f25..92478088a 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -1,4 +1,4 @@ -# Using FreqTrade with Docker +# Using Freqtrade with Docker ## Install Docker @@ -8,13 +8,144 @@ Start by downloading and installing Docker CE for your platform: * [Windows](https://docs.docker.com/docker-for-windows/install/) * [Linux](https://docs.docker.com/install/) +Optionally, [docker-compose](https://docs.docker.com/compose/install/) should be installed and available to follow the [docker quick start guide](#docker-quick-start). + Once you have Docker installed, simply prepare the config file (e.g. `config.json`) and run the image for `freqtrade` as explained below. -## Download the official FreqTrade docker image +## Freqtrade with docker-compose + +Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage. + +!!! Note + The following section assumes that docker and docker-compose is installed and available to the logged in user. + +!!! Note + All below comands use relative directories and will have to be executed from the directory containing the `docker-compose.yml` file. + +!!! Note "Docker on Raspberry" + If you're running freqtrade on a Raspberry PI, you must change the image from `freqtradeorg/freqtrade:master` to `freqtradeorg/freqtrade:master_pi` or `freqtradeorg/freqtrade:develop_pi`, otherwise the image will not work. + +### Docker quick start + +Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory. + +``` bash +mkdir ft_userdata +cd ft_userdata/ +# Download the docker-compose file from the repository +curl https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docker-compose.yml -o docker-compose.yml + +# Pull the freqtrade image +docker-compose pull + +# Create user directory structure +docker-compose run --rm freqtrade create-userdir --userdir user_data + +# Create configuration - Requires answering interactive questions +docker-compose run --rm freqtrade new-config --config user_data/config.json +``` + +The above snippet creates a new directory called "ft_userdata", downloads the latest compose file and pulls the freqtrade image. +The last 2 steps in the snippet create the directory with user-data, as well as (interactively) the default configuration based on your selections. + +!!! Note + You can edit the configuration at any time, which is available as `user_data/config.json` (within the directory `ft_userdata`) when using the above configuration. + +#### Adding your strategy + +The configuration is now available as `user_data/config.json`. +You should now copy your strategy to `user_data/strategies/` - and add the Strategy class name to the `docker-compose.yml` file, replacing `SampleStrategy`. If you wish to run the bot with the SampleStrategy, just leave it as it is. + +!!! Warning + The `SampleStrategy` is there for your reference and give you ideas for your own strategy. + Please always backtest the strategy and use dry-run for some time before risking real money! + +Once this is done, you're ready to launch the bot in trading mode (Dry-run or Live-trading, depending on your answer to the corresponding question you made above). + +``` bash +docker-compose up -d +``` + +#### Docker-compose logs + +Logs will be written to `user_data/logs/freqtrade.log`. +Alternatively, you can check the latest logs using `docker-compose logs -f`. + +#### Database + +The database will be in the user_data directory as well, and will be called `user_data/tradesv3.sqlite`. + +#### Updating freqtrade with docker-compose + +To update freqtrade when using docker-compose is as simple as running the following 2 commands: + +``` bash +# Download the latest image +docker-compose pull +# Restart the image +docker-compose up -d +``` + +This will first pull the latest image, and will then restart the container with the just pulled version. + +!!! Note + You should always check the changelog for breaking changes / manual interventions required and make sure the bot starts correctly after the update. + +#### Going from here + +Advanced users may edit the docker-compose file further to include all possible options or arguments. + +All possible freqtrade arguments will be available by running `docker-compose run --rm freqtrade `. + +!!! Note "`docker-compose run --rm`" + Including `--rm` will clean up the container after completion, and is highly recommended for all modes except trading mode (running with `freqtrade trade` command). + +##### Example: Download data with docker-compose + +Download backtesting data for 5 days for the pair ETH/BTC and 1h timeframe from Binance. The data will be stored in the directory `user_data/data/` on the host. + +``` bash +docker-compose run --rm freqtrade download-data --pairs ETH/BTC --exchange binance --days 5 -t 1h +``` + +Head over to the [Data Downloading Documentation](data-download.md) for more details on downloading data. + +##### Example: Backtest with docker-compose + +Run backtesting in docker-containers for SampleStrategy and specified timerange of historical data, on 5m timeframe: + +``` bash +docker-compose run --rm freqtrade backtesting --config user_data/config.json --strategy SampleStrategy --timerange 20190801-20191001 -i 5m +``` + +Head over to the [Backtesting Documentation](backtesting.md) to learn more. + +#### Additional dependencies with docker-compose + +If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host. +For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) for an example). + +You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions. + +``` yaml + image: freqtrade_custom + build: + context: . + dockerfile: "./Dockerfile." +``` + +You can then run `docker-compose build` to build the docker image, and run it using the commands described above. + +## Freqtrade with docker without docker-compose + +!!! Warning + The below documentation is provided for completeness and assumes that you are somewhat familiar with running docker containers. If you're just starting out with docker, we recommend to follow the [Freqtrade with docker-compose](#freqtrade-with-docker-compose) instructions. + +### Download the official Freqtrade docker image Pull the image from docker hub. -Branches / tags available can be checked out on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). +Branches / tags available can be checked out on [Dockerhub tags page](https://hub.docker.com/r/freqtradeorg/freqtrade/tags/). ```bash docker pull freqtradeorg/freqtrade:develop @@ -164,8 +295,7 @@ docker run -d \ ``` !!! Note - db-url defaults to `sqlite:///tradesv3.sqlite` but it defaults to `sqlite://` if `dry_run=True` is being used. - To override this behaviour use a custom db-url value: i.e.: `--db-url sqlite:///tradesv3.dryrun.sqlite` + When using docker, it's best to specify `--db-url` explicitly to ensure that the database URL and the mounted database file match. !!! Note All available bot command line parameters can be added to the end of the `docker run` command. diff --git a/docs/edge.md b/docs/edge.md index c7b088476..029844c0b 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -1,4 +1,4 @@ -# Edge positioning +# Edge positioning This page explains how to use Edge Positioning module in your bot in order to enter into a trade only if the trade has a reasonable win rate and risk reward ratio, and consequently adjust your position size and stoploss. @@ -9,6 +9,7 @@ This page explains how to use Edge Positioning module in your bot in order to en Edge does not consider anything else than buy/sell/stoploss signals. So trailing stoploss, ROI, and everything else are ignored in its calculation. ## Introduction + Trading is all about probability. No one can claim that he has a strategy working all the time. You have to assume that sometimes you lose. But it doesn't mean there is no rule, it only means rules should work "most of the time". Let's play a game: we toss a coin, heads: I give you 10$, tails: you give me 10$. Is it an interesting game? No, it's quite boring, isn't it? @@ -22,45 +23,63 @@ Let's complicate it more: you win 80% of the time but only 2$, I win 20% of the The question is: How do you calculate that? How do you know if you wanna play? The answer comes to two factors: + - Win Rate - Risk Reward Ratio ### Win Rate + Win Rate (*W*) is is the mean over some amount of trades (*N*) what is the percentage of winning trades to total number of trades (note that we don't consider how much you gained but only if you won or not). - W = (Number of winning trades) / (Total number of trades) = (Number of winning trades) / N +``` +W = (Number of winning trades) / (Total number of trades) = (Number of winning trades) / N +``` Complementary Loss Rate (*L*) is defined as - L = (Number of losing trades) / (Total number of trades) = (Number of losing trades) / N +``` +L = (Number of losing trades) / (Total number of trades) = (Number of losing trades) / N +``` or, which is the same, as - L = 1 – W +``` +L = 1 – W +``` ### Risk Reward Ratio + Risk Reward Ratio (*R*) is a formula used to measure the expected gains of a given investment against the risk of loss. It is basically what you potentially win divided by what you potentially lose: - R = Profit / Loss +``` +R = Profit / Loss +``` Over time, on many trades, you can calculate your risk reward by dividing your average profit on winning trades by your average loss on losing trades: - Average profit = (Sum of profits) / (Number of winning trades) +``` +Average profit = (Sum of profits) / (Number of winning trades) - Average loss = (Sum of losses) / (Number of losing trades) +Average loss = (Sum of losses) / (Number of losing trades) - R = (Average profit) / (Average loss) +R = (Average profit) / (Average loss) +``` ### Expectancy + At this point we can combine *W* and *R* to create an expectancy ratio. This is a simple process of multiplying the risk reward ratio by the percentage of winning trades and subtracting the percentage of losing trades, which is calculated as follows: - Expectancy Ratio = (Risk Reward Ratio X Win Rate) – Loss Rate = (R X W) – L +``` +Expectancy Ratio = (Risk Reward Ratio X Win Rate) – Loss Rate = (R X W) – L +``` So lets say your Win rate is 28% and your Risk Reward Ratio is 5: - Expectancy = (5 X 0.28) – 0.72 = 0.68 +``` +Expectancy = (5 X 0.28) – 0.72 = 0.68 +``` -Superficially, this means that on average you expect this strategy’s trades to return .68 times the size of your loses. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. +Superficially, this means that on average you expect this strategy’s trades to return 1.68 times the size of your loses. Said another way, you can expect to win $1.68 for every $1 you lose. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. It is important to remember that any system with an expectancy greater than 0 is profitable using past data. The key is finding one that will be profitable in the future. @@ -69,6 +88,7 @@ You can also use this value to evaluate the effectiveness of modifications to th **NOTICE:** It's important to keep in mind that Edge is testing your expectancy using historical data, there's no guarantee that you will have a similar edge in the future. It's still vital to do this testing in order to build confidence in your methodology, but be wary of "curve-fitting" your approach to the historical data as things are unlikely to play out the exact same way for future trades. ## How does it work? + If enabled in config, Edge will go through historical data with a range of stoplosses in order to find buy and sell/stoploss signals. It then calculates win rate and expectancy over *N* trades for each stoploss. Here is an example: | Pair | Stoploss | Win Rate | Risk Reward Ratio | Expectancy | @@ -83,6 +103,7 @@ The goal here is to find the best stoploss for the strategy in order to have the Edge module then forces stoploss value it evaluated to your strategy dynamically. ### Position size + Edge also dictates the stake amount for each trade to the bot according to the following factors: - Allowed capital at risk @@ -90,13 +111,17 @@ Edge also dictates the stake amount for each trade to the bot according to the f Allowed capital at risk is calculated as follows: - Allowed capital at risk = (Capital available_percentage) X (Allowed risk per trade) +``` +Allowed capital at risk = (Capital available_percentage) X (Allowed risk per trade) +``` Stoploss is calculated as described above against historical data. Your position size then will be: - Position size = (Allowed capital at risk) / Stoploss +``` +Position size = (Allowed capital at risk) / Stoploss +``` Example: @@ -115,100 +140,30 @@ Available capital doesn’t change before a position is sold. Let’s assume tha So the Bot receives another buy signal for trade 4 with a stoploss at 2% then your position size would be **0.055 / 0.02 = 2.75 ETH**. ## Configurations + Edge module has following configuration options: -#### enabled -If true, then Edge will run periodically. - -(defaults to false) - -#### process_throttle_secs -How often should Edge run in seconds? - -(defaults to 3600 so one hour) - -#### calculate_since_number_of_days -Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy -Note that it downloads historical data so increasing this number would lead to slowing down the bot. - -(defaults to 7) - -#### capital_available_percentage -This is the percentage of the total capital on exchange in stake currency. - -As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital. - -(defaults to 0.5) - -#### allowed_risk -Percentage of allowed risk per trade. - -(defaults to 0.01 so 1%) - -#### stoploss_range_min - -Minimum stoploss. - -(defaults to -0.01) - -#### stoploss_range_max - -Maximum stoploss. - -(defaults to -0.10) - -#### stoploss_range_step - -As an example if this is set to -0.01 then Edge will test the strategy for \[-0.01, -0,02, -0,03 ..., -0.09, -0.10\] ranges. -Note than having a smaller step means having a bigger range which could lead to slow calculation. - -If you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10. - -(defaults to -0.01) - -#### minimum_winrate - -It filters out pairs which don't have at least minimum_winrate. - -This comes handy if you want to be conservative and don't comprise win rate in favour of risk reward ratio. - -(defaults to 0.60) - -#### minimum_expectancy - -It filters out pairs which have the expectancy lower than this number. - -Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return. - -(defaults to 0.20) - -#### min_trade_number - -When calculating *W*, *R* and *E* (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable. - -Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something. - -(defaults to 10, it is highly recommended not to decrease this number) - -#### max_trade_duration_minute - -Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign. - -**NOTICE:** While configuring this value, you should take into consideration your ticker interval. As an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. Default value is set assuming your strategy interval is relatively small (1m or 5m, etc.). - -(defaults to 1 day, i.e. to 60 * 24 = 1440 minutes) - -#### remove_pumps - -Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off. - -(defaults to false) +| Parameter | Description | +|------------|-------------| +| `enabled` | If true, then Edge will run periodically.
*Defaults to `false`.*
**Datatype:** Boolean +| `process_throttle_secs` | How often should Edge run in seconds.
*Defaults to `3600` (once per hour).*
**Datatype:** Integer +| `calculate_since_number_of_days` | Number of days of data against which Edge calculates Win Rate, Risk Reward and Expectancy.
**Note** that it downloads historical data so increasing this number would lead to slowing down the bot.
*Defaults to `7`.*
**Datatype:** Integer +| `capital_available_percentage` | **DEPRECATED - [replaced with `tradable_balance_ratio`](configuration.md#Available balance)** This is the percentage of the total capital on exchange in stake currency.
As an example if you have 10 ETH available in your wallet on the exchange and this value is 0.5 (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers it as available capital.
*Defaults to `0.5`.*
**Datatype:** Float +| `allowed_risk` | Ratio of allowed risk per trade.
*Defaults to `0.01` (1%)).*
**Datatype:** Float +| `stoploss_range_min` | Minimum stoploss.
*Defaults to `-0.01`.*
**Datatype:** Float +| `stoploss_range_max` | Maximum stoploss.
*Defaults to `-0.10`.*
**Datatype:** Float +| `stoploss_range_step` | As an example if this is set to -0.01 then Edge will test the strategy for `[-0.01, -0,02, -0,03 ..., -0.09, -0.10]` ranges.
**Note** than having a smaller step means having a bigger range which could lead to slow calculation.
If you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10.
*Defaults to `-0.001`.*
**Datatype:** Float +| `minimum_winrate` | It filters out pairs which don't have at least minimum_winrate.
This comes handy if you want to be conservative and don't comprise win rate in favour of risk reward ratio.
*Defaults to `0.60`.*
**Datatype:** Float +| `minimum_expectancy` | It filters out pairs which have the expectancy lower than this number.
Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return.
*Defaults to `0.20`.*
**Datatype:** Float +| `min_trade_number` | When calculating *W*, *R* and *E* (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable.
Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
*Defaults to `10` (it is highly recommended not to decrease this number).*
**Datatype:** Integer +| `max_trade_duration_minute` | Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
**NOTICE:** While configuring this value, you should take into consideration your timeframe (ticker interval). As an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. Default value is set assuming your strategy interval is relatively small (1m or 5m, etc.).
*Defaults to `1440` (one day).*
**Datatype:** Integer +| `remove_pumps` | Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
*Defaults to `false`.*
**Datatype:** Boolean ## Running Edge independently You can run Edge independently in order to see in details the result. Here is an example: -```bash +``` bash freqtrade edge ``` diff --git a/docs/exchanges.md b/docs/exchanges.md index 76fa81f4a..06db26f89 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -5,7 +5,7 @@ This page combines common gotchas and informations which are exchange-specific a ## Binance !!! Tip "Stoploss on Exchange" - Binance is currently the only exchange supporting `stoploss_on_exchange`. It provides great advantages, so we recommend to benefit from it. + Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. ### Blacklists @@ -22,6 +22,9 @@ Binance has been split into 3, and users must use the correct ccxt exchange ID f ## Kraken +!!! Tip "Stoploss on Exchange" + Kraken supports `stoploss_on_exchange` and uses stop-loss-market orders. It provides great advantages, so we recommend to benefit from it, however since the resulting order is a stoploss-market order, sell-rates are not guaranteed, which makes this feature less secure than on other exchanges. This limitation is based on kraken's policy [source](https://blog.kraken.com/post/1234/announcement-delisting-pairs-and-temporary-suspension-of-advanced-order-types/) and [source2](https://blog.kraken.com/post/1494/kraken-enables-advanced-orders-and-adds-10-currency-pairs/) - which has stoploss-limit orders disabled. + ### Historic Kraken data The Kraken API does only provide 720 historic candles, which is sufficient for Freqtrade dry-run and live trade modes, but is a problem for backtesting. @@ -29,6 +32,10 @@ To download data for the Kraken exchange, using `--dl-trades` is mandatory, othe ## Bittrex +### Order types + +Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faq.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy). + ### Restricted markets Bittrex split its exchange into US and International versions. @@ -55,6 +62,11 @@ res = [ f"{x['MarketCurrency']}/{x['BaseCurrency']}" for x in ct.publicGetMarket print(res) ``` +## All exchanges + +Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys. + + ## Random notes for other exchanges * The Ocean (exchange id: `theocean`) exchange uses Web3 functionality and requires `web3` python package to be installed: @@ -62,23 +74,13 @@ print(res) $ pip3 install web3 ``` -### Send incomplete candles to the strategy +### Getting latest price / Incomplete candles -Most exchanges return incomplete candles via their ohlcv / klines interface. -By default, Freqtrade assumes that incomplete candles are returned and removes the last candle assuming it's an incomplete candle. +Most exchanges return current incomplete candle via their OHLCV/klines API interface. +By default, Freqtrade assumes that incomplete candle is fetched from the exchange and removes the last candle assuming it's the incomplete candle. Whether your exchange returns incomplete candles or not can be checked using [the helper script](developer.md#Incomplete-candles) from the Contributor documentation. -If the exchange does return incomplete candles and you would like to have incomplete candles in your strategy, you can set the following parameter in the configuration file. +Due to the danger of repainting, Freqtrade does not allow you to use this incomplete candle. -``` json -{ - - "exchange": { - "_ft_has_params": {"ohlcv_partial_candle": false} - } -} -``` - -!!! Warning "Danger of repainting" - Changing this parameter makes the strategy responsible to avoid repainting and handle this accordingly. Doing this is therefore not recommended, and should only be performed by experienced users who are fully aware of the impact this setting has. +However, if it is based on the need for the latest price for your strategy - then this requirement can be acquired using the [data provider](strategy-customization.md#possible-options-for-dataprovider) from within the strategy. diff --git a/docs/faq.md b/docs/faq.md index 2416beae4..8e8a1bf35 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -45,12 +45,28 @@ the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-c You can use the `/forcesell all` command from Telegram. -### I get the message "RESTRICTED_MARKET" +### I'm getting the "RESTRICTED_MARKET" message in the log Currently known to happen for US Bittrex users. Read [the Bittrex section about restricted markets](exchanges.md#restricted-markets) for more information. +### I'm getting the "Exchange Bittrex does not support market orders." message and cannot run my strategy + +As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Probably your strategy was written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex). + +To fix it for Bittrex, redefine order types in the strategy to use "limit" instead of "market": + +``` + order_types = { + ... + 'stoploss': 'limit', + ... + } +``` + +Same fix should be done in the configuration file, if order types are defined in your custom config rather than in the strategy. + ### How do I search the bot logs for something? By default, the bot writes its log into stderr stream. This is implemented this way so that you can easily separate the bot's diagnostics messages from Backtesting, Edge and Hyperopt results, output from other various Freqtrade utility subcommands, as well as from the output of your custom `print()`'s you may have inserted into your strategy. So if you need to search the log messages with the grep utility, you need to redirect stderr to stdout and disregard stdout. @@ -84,7 +100,7 @@ $ tail -f /path/to/mylogfile.log | grep 'something' ``` from a separate terminal window. -On Windows, the `--logfilename` option is also supported by Freqtrade and you can use the `findstr` command to search the log for the string of interest: +On Windows, the `--logfile` option is also supported by Freqtrade and you can use the `findstr` command to search the log for the string of interest: ``` > type \path\to\mylogfile.log | findstr "something" ``` diff --git a/docs/hyperopt.md b/docs/hyperopt.md index f399fe816..11161e58b 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -6,9 +6,7 @@ algorithms included in the `scikit-optimize` package to accomplish this. The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time. -In general, the search for best parameters starts with a few random combinations and then uses Bayesian search with a -ML regressor algorithm (currently ExtraTreesRegressor) to quickly find a combination of parameters in the search hyperspace -that minimizes the value of the [loss function](#loss-functions). +In general, the search for best parameters starts with a few random combinations (see [below](#reproducible-results) for more details) and then uses Bayesian search with a ML regressor algorithm (currently ExtraTreesRegressor) to quickly find a combination of parameters in the search hyperspace that minimizes the value of the [loss function](#loss-functions). Hyperopt requires historic data to be available, just as backtesting does. To learn how to get data for the pairs and exchange you're interested in, head over to the [Data Downloading](data-download.md) section of the documentation. @@ -16,6 +14,24 @@ To learn how to get data for the pairs and exchange you're interested in, head o !!! Bug Hyperopt can crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133) +## Install hyperopt dependencies + +Since Hyperopt dependencies are not needed to run the bot itself, are heavy, can not be easily built on some platforms (like Raspberry PI), they are not installed by default. Before you run Hyperopt, you need to install the corresponding dependencies, as described in this section below. + +!!! Note + Since Hyperopt is a resource intensive process, running it on a Raspberry Pi is not recommended nor supported. + +### Docker + +The docker-image includes hyperopt dependencies, no further action needed. + +### Easy installation script (setup.sh) / Manual installation + +```bash +source .env/bin/activate +pip install -r requirements-hyperopt.txt +``` + ## Prepare Hyperopting Before we start digging into Hyperopt, we recommend you to take a look at @@ -31,9 +47,9 @@ This will create a new hyperopt file from a template, which will be located unde Depending on the space you want to optimize, only some of the below are required: * fill `buy_strategy_generator` - for buy signal optimization -* fill `indicator_space` - for buy signal optimzation +* fill `indicator_space` - for buy signal optimization * fill `sell_strategy_generator` - for sell signal optimization -* fill `sell_indicator_space` - for sell signal optimzation +* fill `sell_indicator_space` - for sell signal optimization !!! Note `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. @@ -47,6 +63,9 @@ Optional - can also be loaded from a strategy: !!! Note Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. +!!! Note + You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods. + Rarely you may also need to override: * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default) @@ -57,12 +76,12 @@ Rarely you may also need to override: !!! Tip "Quickly optimize ROI, stoploss and trailing stoploss" You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything (i.e. without creation of a "complete" Hyperopt class with dimensions, parameters, triggers and guards, as described in this document) from the default hyperopt template by relying on your strategy to do most of the calculations. - ``` python + ```python # Have a working strategy at hand. freqtrade new-hyperopt --hyperopt EmptyHyperopt freqtrade hyperopt --hyperopt EmptyHyperopt --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100 - ``` + ``` ### 1. Install a Custom Hyperopt File @@ -75,17 +94,17 @@ Copy the file `user_data/hyperopts/sample_hyperopt.py` into `user_data/hyperopts There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: -- Inside `indicator_space()` - the parameters hyperopt shall be optimizing. -- Inside `populate_buy_trend()` - applying the parameters. +* Inside `indicator_space()` - the parameters hyperopt shall be optimizing. +* Inside `populate_buy_trend()` - applying the parameters. There you have two different types of indicators: 1. `guards` and 2. `triggers`. 1. Guards are conditions like "never buy if ADX < 10", or never buy if current price is over EMA10. -2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower bollinger band". +2. Triggers are ones that actually trigger buy in specific moment, like "buy when EMA5 crosses over EMA10" or "buy when close price touches lower Bollinger band". Hyperoptimization will, for each eval round, pick one trigger and possibly multiple guards. The constructed strategy will be something like -"*buy exactly when close price touches lower bollinger band, BUT only if +"*buy exactly when close price touches lower Bollinger band, BUT only if ADX > 10*". If you have updated the buy strategy, i.e. changed the contents of @@ -103,9 +122,10 @@ Place the corresponding settings into the following methods The configuration and rules are the same than for buy signals. To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. -#### Using ticker-interval as part of the Strategy +#### Using timeframe as a part of the Strategy -The Strategy exposes the ticker-interval as `self.ticker_interval`. The same value is available as class-attribute `HyperoptName.ticker_interval`. +The Strategy class exposes the timeframe (ticker interval) value as the `self.ticker_interval` attribute. +The same value is available as class-attribute `HyperoptName.ticker_interval`. In the case of the linked sample-value this would be `SampleHyperOpt.ticker_interval`. ## Solving a Mystery @@ -141,7 +161,7 @@ one we call `trigger` and use it to decide which buy trigger we want to use. So let's write the buy strategy using these values: -``` python +```python def populate_buy_trend(dataframe: DataFrame) -> DataFrame: conditions = [] # GUARDS AND TRENDS @@ -159,6 +179,9 @@ So let's write the buy strategy using these values: dataframe['macd'], dataframe['macdsignal'] )) + # Check that volume is not 0 + conditions.append(dataframe['volume'] > 0) + if conditions: dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -172,7 +195,7 @@ So let's write the buy strategy using these values: Hyperopting will now call this `populate_buy_trend` as many times you ask it (`epochs`) with different value combinations. It will then use the given historical data and make buys based on the buy signals generated with the above function and based on the results -it will end with telling you which paramter combination produced the best profits. +it will end with telling you which parameter combination produced the best profits. The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. When you want to test an indicator that isn't used by the bot currently, remember to @@ -182,7 +205,7 @@ add it to the `populate_indicators()` method in your custom hyperopt file. Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results. -By default, FreqTrade uses a loss function, which has been with freqtrade since the beginning and optimizes mostly for short trade duration and avoiding losses. +By default, Freqtrade uses a loss function, which has been with freqtrade since the beginning and optimizes mostly for short trade duration and avoiding losses. A different loss function can be specified by using the `--hyperopt-loss ` argument. This class should be in its own file within the `user_data/hyperopts/` directory. @@ -191,7 +214,10 @@ Currently, the following loss functions are builtin: * `DefaultHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) * `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) -* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on the trade returns) +* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on trade returns relative to standard deviation) +* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on **daily** trade returns relative to standard deviation) +* `SortinoHyperOptLoss` (optimizes Sortino Ratio calculated on trade returns relative to **downside** standard deviation) +* `SortinoHyperOptLossDaily` (optimizes Sortino Ratio calculated on **daily** trade returns relative to **downside** standard deviation) Creation of a custom loss function is covered in the [Advanced Hyperopt](advanced-hyperopt.md) part of the documentation. @@ -206,7 +232,7 @@ We strongly recommend to use `screen` or `tmux` to prevent any connection loss. freqtrade hyperopt --config config.json --hyperopt -e 5000 --spaces all ``` -Use `` as the name of the custom hyperopt used. +Use `` as the name of the custom hyperopt used. The `-e` option will set how many evaluations hyperopt will do. We recommend running at least several thousand evaluations. @@ -219,11 +245,11 @@ The `--spaces all` option determines that all possible parameters should be opti !!! Warning When switching parameters or changing configuration options, make sure to not use the argument `--continue` so temporary results can be removed. -### Execute Hyperopt with Different Ticker-Data Source +### Execute Hyperopt with different historical data source -If you would like to hyperopt parameters using an alternate ticker data that -you have on-disk, use the `--datadir PATH` option. Default hyperopt will -use data from directory `user_data/data`. +If you would like to hyperopt parameters using an alternate historical data set that +you have on-disk, use the `--datadir PATH` option. By default, hyperopt +uses data from directory `user_data/data`. ### Running Hyperopt with Smaller Testset @@ -265,28 +291,28 @@ The default Hyperopt Search Space, used when no `--space` command line option is ### Position stacking and disabling max market positions -In some situations, you may need to run Hyperopt (and Backtesting) with the +In some situations, you may need to run Hyperopt (and Backtesting) with the `--eps`/`--enable-position-staking` and `--dmmp`/`--disable-max-market-positions` arguments. By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one -open trade is allowed for every traded pair. The total number of trades open for all pairs +open trade is allowed for every traded pair. The total number of trades open for all pairs is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to -some potential trades to be hidden (or masked) by previosly open trades. +some potential trades to be hidden (or masked) by previously open trades. The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times, -while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades` +while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades` during Hyperopt/Backtesting (which is equal to setting `max_open_trades` to a very high number). !!! Note Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality. -You can also enable position stacking in the configuration file by explicitly setting +You can also enable position stacking in the configuration file by explicitly setting `"position_stacking"=true`. ### Reproducible results -The search for optimal parameters starts with a few (currently 30) random combinations in the hyperspace of parameters, random Hyperopt epochs. These random epochs are marked with a leading asterisk sign at the Hyperopt output. +The search for optimal parameters starts with a few (currently 30) random combinations in the hyperspace of parameters, random Hyperopt epochs. These random epochs are marked with an asterisk character (`*`) in the first column in the Hyperopt output. The initial state for generation of these random values (random state) is controlled by the value of the `--random-state` command line option. You can set it to some arbitrary value of your choice to obtain reproducible results. @@ -323,7 +349,7 @@ method, what those values match to. So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block: -``` python +```python (dataframe['rsi'] < 29.0) ``` @@ -372,20 +398,21 @@ In order to use this best ROI table found by Hyperopt in backtesting and for liv 118: 0 } ``` + As stated in the comment, you can also use it as the value of the `minimal_roi` setting in the configuration file. #### Default ROI Search Space -If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the ticker_interval used. By default the values vary in the following ranges (for some of the most used ticker intervals, values are rounded to 5 digits after the decimal point): +If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the ticker_interval used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 5 digits after the decimal point): -| # step | 1m | | 5m | | 1h | | 1d | | -|---|---|---|---|---|---|---|---|---| -| 1 | 0 | 0.01161...0.11992 | 0 | 0.03...0.31 | 0 | 0.06883...0.71124 | 0 | 0.12178...1.25835 | -| 2 | 2...8 | 0.00774...0.04255 | 10...40 | 0.02...0.11 | 120...480 | 0.04589...0.25238 | 2880...11520 | 0.08118...0.44651 | -| 3 | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 | -| 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 | +| # step | 1m | | 5m | | 1h | | 1d | | +| ------ | ------ | ----------------- | -------- | ----------- | ---------- | ----------------- | ------------ | ----------------- | +| 1 | 0 | 0.01161...0.11992 | 0 | 0.03...0.31 | 0 | 0.06883...0.71124 | 0 | 0.12178...1.25835 | +| 2 | 2...8 | 0.00774...0.04255 | 10...40 | 0.02...0.11 | 120...480 | 0.04589...0.25238 | 2880...11520 | 0.08118...0.44651 | +| 3 | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 | +| 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 | -These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the ticker interval used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the ticker interval used. +These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe (ticker interval) used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used. If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default. @@ -416,6 +443,7 @@ In order to use this best stoploss value found by Hyperopt in backtesting and fo # This attribute will be overridden if the config file contains "stoploss" stoploss = -0.27996 ``` + As stated in the comment, you can also use it as the value of the `stoploss` setting in the configuration file. #### Default Stoploss Search Space @@ -452,6 +480,7 @@ In order to use these best trailing stop parameters found by Hyperopt in backtes trailing_stop_positive_offset = 0.06038 trailing_only_offset_is_reached = True ``` + As stated in the comment, you can also use it as the values of the corresponding settings in the configuration file. #### Default Trailing Stop Search Space diff --git a/docs/index.md b/docs/index.md index 206d635c6..adc661300 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,5 @@ # Freqtrade -[![Build Status](https://travis-ci.org/freqtrade/freqtrade.svg?branch=develop)](https://travis-ci.org/freqtrade/freqtrade) +[![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/) [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop) [![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability) @@ -11,8 +11,10 @@ Download Follow @freqtrade + ## Introduction -Freqtrade is a cryptocurrency trading bot written in Python. + +Freqtrade is a crypto-currency algorithmic trading software developed in python (3.6+) and supported on Windows, macOS and Linux. !!! Danger "DISCLAIMER" This software is for educational purposes only. Do not risk money which you are afraid to lose. USE THE SOFTWARE AT YOUR OWN RISK. THE AUTHORS AND ALL AFFILIATES ASSUME NO RESPONSIBILITY FOR YOUR TRADING RESULTS. @@ -23,18 +25,15 @@ Freqtrade is a cryptocurrency trading bot written in Python. ## Features - - Based on Python 3.6+: For botting on any operating system — Windows, macOS and Linux. - - Persistence: Persistence is achieved through sqlite database. - - Dry-run mode: Run the bot without playing money. - - Backtesting: Run a simulation of your buy/sell strategy with historical data. - - Strategy Optimization by machine learning: Use machine learning to optimize your buy/sell strategy parameters with real exchange data. - - Edge position sizing: Calculate your win rate, risk reward ratio, the best stoploss and adjust your position size before taking a position for each specific market. - - Whitelist crypto-currencies: Select which crypto-currency you want to trade or use dynamic whitelists based on market (pair) trade volume. - - Blacklist crypto-currencies: Select which crypto-currency you want to avoid. - - Manageable via Telegram or REST APi: Manage the bot with Telegram or via the builtin REST API. - - Display profit/loss in fiat: Display your profit/loss in any of 33 fiat currencies supported. - - Daily summary of profit/loss: Receive the daily summary of your profit/loss. - - Performance status report: Receive the performance status of your current trades. +- Develop your Strategy: Write your strategy in python, using [pandas](https://pandas.pydata.org/). Example strategies to inspire you are available in the [strategy repository](https://github.com/freqtrade/freqtrade-strategies). +- Download market data: Download historical data of the exchange and the markets your may want to trade with. +- Backtest: Test your strategy on downloaded historical data. +- Optimize: Find the best parameters for your strategy using hyperoptimization which employs machining learning methods. You can optimize buy, sell, take profit (ROI), stop-loss and trailing stop-loss parameters for your strategy. +- Select markets: Create your static list or use an automatic one based on top traded volumes and/or prices (not available during backtesting). You can also explicitly blacklist markets you don't want to trade. +- Run: Test your strategy with simulated money (Dry-Run mode) or deploy it with real money (Live-Trade mode). +- Run using Edge (optional module): The concept is to find the best historical [trade expectancy](edge.md#expectancy) by markets based on variation of the stop-loss and then allow/reject markets to trade. The sizing of the trade is based on a risk of a percentage of your capital. +- Control/Monitor: Use Telegram or a REST API (start/stop the bot, show profit/loss, daily summary, current open trades results, etc.). +- Analyse: Further analysis can be performed on either Backtesting data or Freqtrade trading history (SQL database), including automated standard plots, and methods to load the data into [interactive environments](data-analysis.md). ## Requirements @@ -52,20 +51,23 @@ To run this bot we recommend you a cloud instance with a minimum of: ### Software requirements +- Docker (Recommended) + +Alternatively + - Python 3.6.x - pip (pip3) - git - TA-Lib - virtualenv (Recommended) -- Docker (Recommended) ## Support -Help / Slack -For any questions not covered by the documentation or for further information about the bot, we encourage you to join our Slack channel. +### Help / Slack +For any questions not covered by the documentation or for further information about the bot, we encourage you to join our passionate Slack community. -Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) to join Slack channel. +Click [here](https://join.slack.com/t/highfrequencybot/shared_invite/enQtNjU5ODcwNjI1MDU3LTU1MTgxMjkzNmYxNWE1MDEzYzQ3YmU4N2MwZjUyNjJjODRkMDVkNjg4YTAyZGYzYzlhOTZiMTE4ZjQ4YzM0OGE) to join the Freqtrade Slack channel. ## Ready to try? -Begin by reading our installation guide [here](installation). +Begin by reading our installation guide [for docker](docker.md), or for [installation without docker](installation.md). diff --git a/docs/installation.md b/docs/installation.md index 27b7a94c5..f017bef96 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,6 +2,8 @@ This page explains how to prepare your environment for running the bot. +Please consider using the prebuilt [docker images](docker.md) to get started quickly while trying out freqtrade evaluating how it operates. + ## Prerequisite ### Requirements @@ -14,15 +16,7 @@ Click each one for install guide: * [virtualenv](https://virtualenv.pypa.io/en/stable/installation/) (Recommended) * [TA-Lib](https://mrjbq7.github.io/ta-lib/install.html) (install instructions below) -### API keys - -Before running your bot in production you will need to setup few -external API. In production mode, the bot will require valid Exchange API -credentials. We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot) (optional but recommended). - -### Setup your exchange account - -You will need to create API Keys (Usually you get `key` and `secret`) from the Exchange website and insert this into the appropriate fields in the configuration or when asked by the installation script. + We also recommend a [Telegram bot](telegram-usage.md#setup-your-telegram-bot), which is optional but recommended. ## Quick start @@ -31,7 +25,7 @@ Freqtrade provides the Linux/MacOS Easy Installation script to install all depen !!! Note Windows installation is explained [here](#windows). -The easiest way to install and run Freqtrade is to clone the bot GitHub repository and then run the Easy Installation script, if it's available for your platform. +The easiest way to install and run Freqtrade is to clone the bot Github repository and then run the Easy Installation script, if it's available for your platform. !!! Note "Version considerations" When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `master` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). @@ -42,11 +36,12 @@ The easiest way to install and run Freqtrade is to clone the bot GitHub reposito This can be achieved with the following commands: ```bash -git clone git@github.com:freqtrade/freqtrade.git +git clone https://github.com/freqtrade/freqtrade.git cd freqtrade git checkout master # Optional, see (1) ./setup.sh --install ``` + (1) This command switches the cloned repository to the use of the `master` branch. It's not needed if you wish to stay on the `develop` branch. You may later switch between branches at any time with the `git checkout master`/`git checkout develop` commands. ## Easy Installation Script (Linux/MacOS) @@ -64,11 +59,11 @@ usage: ** --install ** -With this option, the script will install everything you need to run the bot: +With this option, the script will install the bot and most dependencies: +You will need to have git and python3.6+ installed beforehand for this to work. * Mandatory software as: `ta-lib` -* Setup your virtualenv -* Configure your `config.json` file +* Setup your virtualenv under `.env/` This option is a combination of installation tasks, `--reset` and `--config`. @@ -82,7 +77,7 @@ This option will hard reset your branch (only if you are on either `master` or ` ** --config ** -Use this option to configure the `config.json` configuration file. The script will interactively ask you questions to setup your bot and create your `config.json`. +DEPRECATED - use `freqtrade new-config -c config.json` instead. ------ @@ -129,6 +124,17 @@ bash setup.sh -i #### 1. Install TA-Lib +Use the provided ta-lib installation script + +```bash +sudo ./build_helpers/install_ta-lib.sh +``` + +!!! Note + This will use the ta-lib tar.gz included in this repository. + +##### TA-Lib manual installation + Official webpage: https://mrjbq7.github.io/ta-lib/install.html ```bash @@ -184,7 +190,8 @@ python3 -m pip install -e . # Initialize the user_directory freqtrade create-userdir --userdir user_data/ -cp config.json.example config.json +# Create a new configuration file +freqtrade new-config --config config.json ``` > *To edit the config please refer to [Bot Configuration](configuration.md).* @@ -241,14 +248,14 @@ git clone https://github.com/freqtrade/freqtrade.git Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl` (make sure to use the version matching your python version) +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial precompiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version) ```cmd >cd \path\freqtrade-develop >python -m venv .env >.env\Scripts\activate.bat REM optionally install ta-lib from wheel -REM >pip install TA_Lib‑0.4.17‑cp36‑cp36m‑win32.whl +REM >pip install TA_Lib‑0.4.18‑cp38‑cp38‑win_amd64.whl >pip install -r requirements.txt >pip install -e . >freqtrade @@ -270,3 +277,18 @@ The easiest way is to download install Microsoft Visual Studio Community [here]( Now you have an environment ready, the next step is [Bot Configuration](configuration.md). + +## Troubleshooting + +### MacOS installation error + +Newer versions of MacOS may have installation failed with errors like `error: command 'g++' failed with exit status 1`. + +This error will require explicit installation of the SDK Headers, which are not installed by default in this version of MacOS. +For MacOS 10.14, this can be accomplished with the below command. + +``` bash +open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg +``` + +If this file is inexistant, then you're probably on a different version of MacOS, so you may need to consult the internet for specific resolution details. diff --git a/docs/plotting.md b/docs/plotting.md index 982a5cd65..be83065a6 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -32,6 +32,7 @@ usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--trade-source {DB,file}] [--export EXPORT] [--export-filename PATH] [--timerange TIMERANGE] [-i TICKER_INTERVAL] + [--no-trades] optional arguments: -h, --help show this help message and exit @@ -50,32 +51,36 @@ optional arguments: values cause huge files. Default: 750. --db-url PATH Override trades database URL, this is useful in custom deployments (default: `sqlite:///tradesv3.sqlite` for - Live Run mode, `sqlite://` for Dry Run). + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). --trade-source {DB,file} Specify the source for trades (Can be DB or file (backtest file)) Default: file --export EXPORT Export backtest results, argument are: trades. Example: `--export=trades` --export-filename PATH - Save backtest results to the file with this filename - (default: `user_data/backtest_results/backtest- - result.json`). Requires `--export` to be set as well. - Example: `--export-filename=user_data/backtest_results - /backtest_today.json` + Save backtest results to the file with this filename. + Requires `--export` to be set as well. Example: + `--export-filename=user_data/backtest_results/backtest + _today.json` --timerange TIMERANGE Specify what timerange of data to use. -i TICKER_INTERVAL, --ticker-interval TICKER_INTERVAL Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`). + --no-trades Skip using trades from backtesting file and DB. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. -d PATH, --datadir PATH Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH @@ -83,10 +88,9 @@ Common arguments: Strategy arguments: -s NAME, --strategy NAME - Specify strategy class name (default: - `DefaultStrategy`). + Specify strategy class name which will be used by the + bot. --strategy-path PATH Specify additional strategy lookup path. - ``` Example: @@ -136,21 +140,83 @@ To plot trades from a backtesting result, use `--export-filename ` freqtrade plot-dataframe --strategy AwesomeStrategy --export-filename user_data/backtest_results/backtest-result.json -p BTC/ETH ``` +### Plot dataframe basics + +![plot-dataframe2](assets/plot-dataframe2.png) + +The `plot-dataframe` subcommand requires backtesting data, a strategy and either a backtesting-results file or a database, containing trades corresponding to the strategy. + +The resulting plot will have the following elements: + +* Green triangles: Buy signals from the strategy. (Note: not every buy signal generates a trade, compare to cyan circles.) +* Red triangles: Sell signals from the strategy. (Also, not every sell signal terminates a trade, compare to red and green squares.) +* Cyan circles: Trade entry points. +* Red squares: Trade exit points for trades with loss or 0% profit. +* Green squares: Trade exit points for profitable trades. +* Indicators with values corresponding to the candle scale (e.g. SMA/EMA), as specified with `--indicators1`. +* Volume (bar chart at the bottom of the main chart). +* Indicators with values in different scales (e.g. MACD, RSI) below the volume bars, as specified with `--indicators2`. + +!!! Note "Bollinger Bands" + Bollinger bands are automatically added to the plot if the columns `bb_lowerband` and `bb_upperband` exist, and are painted as a light blue area spanning from the lower band to the upper band. + +#### Advanced plot configuration + +An advanced plot configuration can be specified in the strategy in the `plot_config` parameter. + +Additional features when using plot_config include: + +* Specify colors per indicator +* Specify additional subplots + +The sample plot configuration below specifies fixed colors for the indicators. Otherwise consecutive plots may produce different colorschemes each time, making comparisons difficult. +It also allows multiple subplots to display both MACD and RSI at the same time. + +Sample configuration with inline comments explaining the process: + +``` python + plot_config = { + 'main_plot': { + # Configuration for main plot indicators. + # Specifies `ema10` to be red, and `ema50` to be a shade of gray + 'ema10': {'color': 'red'}, + 'ema50': {'color': '#CCCCCC'}, + # By omitting color, a random color is selected. + 'sar': {}, + }, + 'subplots': { + # Create subplot MACD + "MACD": { + 'macd': {'color': 'blue'}, + 'macdsignal': {'color': 'orange'}, + }, + # Additional subplot RSI + "RSI": { + 'rsi': {'color': 'red'}, + } + } + } +``` + +!!! Note + The above configuration assumes that `ema10`, `ema50`, `macd`, `macdsignal` and `rsi` are columns in the DataFrame created by the strategy. + ## Plot profit ![plot-profit](assets/plot-profit.png) -The `freqtrade plot-profit` subcommand shows an interactive graph with three plots: +The `plot-profit` subcommand shows an interactive graph with three plots: -1) Average closing price for all pairs -2) The summarized profit made by backtesting. - Note that this is not the real-world profit, but more of an estimate. -3) Profit for each individual pair +* Average closing price for all pairs. +* The summarized profit made by backtesting. +Note that this is not the real-world profit, but more of an estimate. +* Profit for each individual pair. The first graph is good to get a grip of how the overall market progresses. The second graph will show if your algorithm works or doesn't. Perhaps you want an algorithm that steadily makes small profits, or one that acts less often, but makes big swings. +This graph will also highlight the start (and end) of the Max drawdown period. The third graph can be useful to spot outliers, events in pairs that cause profit spikes. @@ -173,14 +239,14 @@ optional arguments: --export EXPORT Export backtest results, argument are: trades. Example: `--export=trades` --export-filename PATH - Save backtest results to the file with this filename - (default: `user_data/backtest_results/backtest- - result.json`). Requires `--export` to be set as well. - Example: `--export-filename=user_data/backtest_results - /backtest_today.json` + Save backtest results to the file with this filename. + Requires `--export` to be set as well. Example: + `--export-filename=user_data/backtest_results/backtest + _today.json` --db-url PATH Override trades database URL, this is useful in custom deployments (default: `sqlite:///tradesv3.sqlite` for - Live Run mode, `sqlite://` for Dry Run). + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). --trade-source {DB,file} Specify the source for trades (Can be DB or file (backtest file)) Default: file @@ -190,7 +256,9 @@ optional arguments: Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. -V, --version show program's version number and exit -c PATH, --config PATH Specify configuration file (default: `config.json`). diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 3e53c15e3..485cd50cf 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,2 +1,2 @@ -mkdocs-material==4.6.0 +mkdocs-material==5.1.7 mdx_truly_sane_lists==1.2 diff --git a/docs/rest-api.md b/docs/rest-api.md index 187a71c97..7f1a95b12 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -11,6 +11,7 @@ Sample configuration: "enabled": true, "listen_ip_address": "127.0.0.1", "listen_port": 8080, + "jwt_secret_key": "somethingrandom", "username": "Freqtrader", "password": "SuperSecret1!" }, @@ -29,7 +30,7 @@ This should return the response: {"status":"pong"} ``` -All other endpoints return sensitive info and require authentication, so are not available through a web browser. +All other endpoints return sensitive info and require authentication and are therefore not available through a web browser. To generate a secure password, either use a password manager, or use the below code snipped. @@ -38,6 +39,9 @@ import secrets secrets.token_hex() ``` +!!! Hint + Use the same method to also generate a JWT secret key (`jwt_secret_key`). + ### Configuration with docker If you run your bot using docker, you'll need to have the bot listen to incomming connections. The security is then handled by docker. @@ -74,7 +78,7 @@ docker run -d \ ## Consuming the API You can consume the API by using the script `scripts/rest_client.py`. -The client script only requires the `requests` module, so FreqTrade does not need to be installed on the system. +The client script only requires the `requests` module, so Freqtrade does not need to be installed on the system. ``` bash python3 scripts/rest_client.py [optional parameters] @@ -202,3 +206,28 @@ whitelist Show the current whitelist :returns: json object ``` + +## Advanced API usage using JWT tokens + +!!! Note + The below should be done in an application (a Freqtrade REST API client, which fetches info via API), and is not intended to be used on a regular basis. + +Freqtrade's REST API also offers JWT (JSON Web Tokens). +You can login using the following command, and subsequently use the resulting access_token. + +``` bash +> curl -X POST --user Freqtrader http://localhost:8080/api/v1/token/login +{"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiMmEwYmY0NWUtMjhmOS00YTUzLTlmNzItMmM5ZWVlYThkNzc2IiwiZXhwIjoxNTg5MTIwNTgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.qt6MAXYIa-l556OM7arBvYJ0SDI9J8bIk3_glDujF5g","refresh_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiZWQ1ZWI3YjAtYjMwMy00YzAyLTg2N2MtNWViMjIxNWQ2YTMxIiwiZXhwIjoxNTkxNzExNjgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJ0eXBlIjoicmVmcmVzaCJ9.d1AT_jYICyTAjD0fiQAr52rkRqtxCjUGEMwlNuuzgNQ"} + +> access_token="eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk2ODEsIm5iZiI6MTU4OTExOTY4MSwianRpIjoiMmEwYmY0NWUtMjhmOS00YTUzLTlmNzItMmM5ZWVlYThkNzc2IiwiZXhwIjoxNTg5MTIwNTgxLCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.qt6MAXYIa-l556OM7arBvYJ0SDI9J8bIk3_glDujF5g" +# Use access_token for authentication +> curl -X GET --header "Authorization: Bearer ${access_token}" http://localhost:8080/api/v1/count + +``` + +Since the access token has a short timeout (15 min) - the `token/refresh` request should be used periodically to get a fresh access token: + +``` bash +> curl -X POST --header "Authorization: Bearer ${refresh_token}"http://localhost:8080/api/v1/token/refresh +{"access_token":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE1ODkxMTk5NzQsIm5iZiI6MTU4OTExOTk3NCwianRpIjoiMDBjNTlhMWUtMjBmYS00ZTk0LTliZjAtNWQwNTg2MTdiZDIyIiwiZXhwIjoxNTg5MTIwODc0LCJpZGVudGl0eSI6eyJ1IjoiRnJlcXRyYWRlciJ9LCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.1seHlII3WprjjclY6DpRhen0rqdF4j6jbvxIhUFaSbs"} +``` diff --git a/docs/sql_cheatsheet.md b/docs/sql_cheatsheet.md index f41520bd9..895a0536a 100644 --- a/docs/sql_cheatsheet.md +++ b/docs/sql_cheatsheet.md @@ -1,13 +1,20 @@ # SQL Helper + This page contains some help if you want to edit your sqlite db. ## Install sqlite3 -**Ubuntu/Debian installation** + +Sqlite3 is a terminal based sqlite application. +Feel free to use a visual Database editor like SqliteBrowser if you feel more comfortable with that. + +### Ubuntu/Debian installation + ```bash sudo apt-get install sqlite3 ``` ## Open the DB + ```bash sqlite3 .open @@ -16,45 +23,61 @@ sqlite3 ## Table structure ### List tables + ```bash .tables ``` ### Display table structure + ```bash .schema ``` ### Trade table structure + ```sql -CREATE TABLE trades ( - id INTEGER NOT NULL, - exchange VARCHAR NOT NULL, - pair VARCHAR NOT NULL, - is_open BOOLEAN NOT NULL, - fee_open FLOAT NOT NULL, - fee_close FLOAT NOT NULL, - open_rate FLOAT, - open_rate_requested FLOAT, - close_rate FLOAT, - close_rate_requested FLOAT, - close_profit FLOAT, - stake_amount FLOAT NOT NULL, - amount FLOAT, - open_date DATETIME NOT NULL, - close_date DATETIME, - open_order_id VARCHAR, - stop_loss FLOAT, - initial_stop_loss FLOAT, - stoploss_order_id VARCHAR, - stoploss_last_update DATETIME, - max_rate FLOAT, - sell_reason VARCHAR, - strategy VARCHAR, - ticker_interval INTEGER, - PRIMARY KEY (id), - CHECK (is_open IN (0, 1)) +CREATE TABLE trades + id INTEGER NOT NULL, + exchange VARCHAR NOT NULL, + pair VARCHAR NOT NULL, + is_open BOOLEAN NOT NULL, + fee_open FLOAT NOT NULL, + fee_open_cost FLOAT, + fee_open_currency VARCHAR, + fee_close FLOAT NOT NULL, + fee_close_cost FLOAT, + fee_close_currency VARCHAR, + open_rate FLOAT, + open_rate_requested FLOAT, + open_trade_price FLOAT, + close_rate FLOAT, + close_rate_requested FLOAT, + close_profit FLOAT, + close_profit_abs FLOAT, + stake_amount FLOAT NOT NULL, + amount FLOAT, + open_date DATETIME NOT NULL, + close_date DATETIME, + open_order_id VARCHAR, + stop_loss FLOAT, + stop_loss_pct FLOAT, + initial_stop_loss FLOAT, + initial_stop_loss_pct FLOAT, + stoploss_order_id VARCHAR, + stoploss_last_update DATETIME, + max_rate FLOAT, + min_rate FLOAT, + sell_reason VARCHAR, + strategy VARCHAR, + ticker_interval INTEGER, + PRIMARY KEY (id), + CHECK (is_open IN (0, 1)) ); +CREATE INDEX ix_trades_stoploss_order_id ON trades (stoploss_order_id); +CREATE INDEX ix_trades_pair ON trades (pair); +CREATE INDEX ix_trades_is_open ON trades (is_open); + ``` ## Get all trades in the table @@ -67,22 +90,32 @@ SELECT * FROM trades; !!! Warning Manually selling a pair on the exchange will not be detected by the bot and it will try to sell anyway. Whenever possible, forcesell should be used to accomplish the same thing. - It is strongly advised to backup your database file before making any manual changes. + It is strongly advised to backup your database file before making any manual changes. !!! Note This should not be necessary after /forcesell, as forcesell orders are closed automatically by the bot on the next iteration. ```sql UPDATE trades -SET is_open=0, close_date=, close_rate=, close_profit=close_rate/open_rate-1, sell_reason= +SET is_open=0, + close_date=, + close_rate=, + close_profit=close_rate/open_rate-1, + close_profit_abs = (amount * * (1 - fee_close) - (amount * open_rate * 1 - fee_open), + sell_reason= WHERE id=; ``` -##### Example +### Example ```sql UPDATE trades -SET is_open=0, close_date='2017-12-20 03:08:45.103418', close_rate=0.19638016, close_profit=0.0496, sell_reason='force_sell' +SET is_open=0, + close_date='2017-12-20 03:08:45.103418', + close_rate=0.19638016, + close_profit=0.0496, + close_profit_abs = (amount * 0.19638016 * (1 - fee_close) - (amount * open_rate * 1 - fee_open) + sell_reason='force_sell' WHERE id=31; ``` @@ -99,10 +132,3 @@ VALUES ('bittrex', 'ETH/BTC', 1, 0.0025, 0.0025, , , bool: + if trade.open_rate > 100 and trade.open_date < datetime.utcnow() - timedelta(minutes=5): + return True + elif trade.open_rate > 10 and trade.open_date < datetime.utcnow() - timedelta(minutes=3): + return True + elif trade.open_rate < 1 and trade.open_date < datetime.utcnow() - timedelta(hours=24): + return True + return False + + + def check_sell_timeout(self, pair: str, trade: 'Trade', order: dict, **kwargs) -> bool: + if trade.open_rate > 100 and trade.open_date < datetime.utcnow() - timedelta(minutes=5): + return True + elif trade.open_rate > 10 and trade.open_date < datetime.utcnow() - timedelta(minutes=3): + return True + elif trade.open_rate < 1 and trade.open_date < datetime.utcnow() - timedelta(hours=24): + return True + return False +``` + +!!! Note + For the above example, `unfilledtimeout` must be set to something bigger than 24h, otherwise that type of timeout will apply first. + +### Custom order timeout example (using additional data) + +``` python +from datetime import datetime +from freqtrade.persistence import Trade + +class Awesomestrategy(IStrategy): + + # ... populate_* methods + + # Set unfilledtimeout to 25 hours, since our maximum timeout from below is 24 hours. + unfilledtimeout = { + 'buy': 60 * 25, + 'sell': 60 * 25 + } + + def check_buy_timeout(self, pair: str, trade: Trade, order: dict, **kwargs) -> bool: + ob = self.dp.orderbook(pair, 1) + current_price = ob['bids'][0][0] + # Cancel buy order if price is more than 2% above the order. + if current_price > order['price'] * 1.02: + return True + return False + + + def check_sell_timeout(self, pair: str, trade: Trade, order: dict, **kwargs) -> bool: + ob = self.dp.orderbook(pair, 1) + current_price = ob['asks'][0][0] + # Cancel sell order if price is more than 2% below the order. + if current_price < order['price'] * 0.98: + return True + return False +``` diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index 4efca7d2f..7197b0fba 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -1,7 +1,6 @@ # Strategy Customization -This page explains where to customize your strategies, and add new -indicators. +This page explains where to customize your strategies, and add new indicators. ## Install a custom strategy file @@ -84,7 +83,7 @@ def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame Performance Note: For the best performance be frugal on the number of indicators you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param dataframe: Dataframe with data from the exchange :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ @@ -249,6 +248,23 @@ minimal_roi = { While technically not completely disabled, this would sell once the trade reaches 10000% Profit. +To use times based on candle duration (ticker_interval or timeframe), the following snippet can be handy. +This will allow you to change the ticket_interval for the strategy, and ROI times will still be set as candles (e.g. after 3 candles ...) + +``` python +from freqtrade.exchange import timeframe_to_minutes + +class AwesomeStrategy(IStrategy): + + ticker_interval = "1d" + ticker_interval_mins = timeframe_to_minutes(ticker_interval) + minimal_roi = { + "0": 0.05, # 5% for the first 3 candles + str(ticker_interval_mins * 3)): 0.02, # 2% after 3 candles + str(ticker_interval_mins * 6)): 0.01, # 1% After 6 candles + } +``` + ### Stoploss Setting a stoploss is highly recommended to protect your capital from strong moves against you. @@ -267,13 +283,14 @@ If your exchange supports it, it's recommended to also set `"stoploss_on_exchang For more information on order_types please look [here](configuration.md#understand-order_types). -### Ticker interval +### Timeframe (ticker interval) This is the set of candles the bot should download and use for the analysis. Common values are `"1m"`, `"5m"`, `"15m"`, `"1h"`, however all values supported by your exchange should work. -Please note that the same buy/sell signals may work with one interval, but not the other. -This setting is accessible within the strategy by using `self.ticker_interval`. +Please note that the same buy/sell signals may work well with one timeframe, but not with the others. + +This setting is accessible within the strategy methods as the `self.ticker_interval` attribute. ### Metadata dict @@ -307,67 +324,14 @@ class Awesomestrategy(IStrategy): !!! Note If the data is pair-specific, make sure to use pair as one of the keys in the dictionary. -### Additional data (DataProvider) +*** -The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. - -All methods return `None` in case of failure (do not raise an exception). - -Please always check the mode of operation to select the correct method to get data (samples see below). - -#### Possible options for DataProvider - -- `available_pairs` - Property with tuples listing cached pairs with their intervals (pair, interval). -- `ohlcv(pair, timeframe)` - Currently cached ticker data for the pair, returns DataFrame or empty DataFrame. -- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. -- `get_pair_dataframe(pair, timeframe)` - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). -- `orderbook(pair, maximum)` - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. -- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on Market data structure. -- `runmode` - Property containing the current runmode. - -#### Example: fetch live ohlcv / historic data for the first informative pair - -``` python -if self.dp: - inf_pair, inf_timeframe = self.informative_pairs()[0] - informative = self.dp.get_pair_dataframe(pair=inf_pair, - timeframe=inf_timeframe) -``` - -!!! Warning "Warning about backtesting" - Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` - for the backtesting runmode) provides the full time-range in one go, - so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). - -!!! Warning "Warning in hyperopt" - This option cannot currently be used during hyperopt. - -#### Orderbook - -``` python -if self.dp: - if self.dp.runmode in ('live', 'dry_run'): - ob = self.dp.orderbook(metadata['pair'], 1) - dataframe['best_bid'] = ob['bids'][0][0] - dataframe['best_ask'] = ob['asks'][0][0] -``` - -!!! Warning - The order book is not part of the historic data which means backtesting and hyperopt will not work if this - method is used. - -#### Available Pairs - -``` python -if self.dp: - for pair, ticker in self.dp.available_pairs: - print(f"available {pair}, {ticker}") -``` +### Additional data (informative_pairs) #### Get data for non-tradeable pairs Data for additional, informative pairs (reference pairs) can be beneficial for some strategies. -Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see above). +Ohlcv data for these pairs will be downloaded as part of the regular whitelist refresh process and is available via `DataProvider` just as other pairs (see below). These parts will **not** be traded unless they are also specified in the pair whitelist, or have been selected by Dynamic Whitelisting. The pairs need to be specified as tuples in the format `("pair", "interval")`, with pair as the first and time interval as the second argument. @@ -387,6 +351,125 @@ def informative_pairs(self): It is however better to use resampling to longer time-intervals when possible to avoid hammering the exchange with too many requests and risk being blocked. +*** + +### Additional data (DataProvider) + +The strategy provides access to the `DataProvider`. This allows you to get additional data to use in your strategy. + +All methods return `None` in case of failure (do not raise an exception). + +Please always check the mode of operation to select the correct method to get data (samples see below). + +#### Possible options for DataProvider + +- [`available_pairs`](#available_pairs) - Property with tuples listing cached pairs with their intervals (pair, interval). +- [`current_whitelist()`](#current_whitelist) - Returns a current list of whitelisted pairs. Useful for accessing dynamic whitelists (ie. VolumePairlist) +- [`get_pair_dataframe(pair, timeframe)`](#get_pair_dataframepair-timeframe) - This is a universal method, which returns either historical data (for backtesting) or cached live data (for the Dry-Run and Live-Run modes). +- `historic_ohlcv(pair, timeframe)` - Returns historical data stored on disk. +- `market(pair)` - Returns market data for the pair: fees, limits, precisions, activity flag, etc. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#markets) for more details on the Market data structure. +- `ohlcv(pair, timeframe)` - Currently cached candle (OHLCV) data for the pair, returns DataFrame or empty DataFrame. +- [`orderbook(pair, maximum)`](#orderbookpair-maximum) - Returns latest orderbook data for the pair, a dict with bids/asks with a total of `maximum` entries. +- [`ticker(pair)`](#tickerpair) - Returns current ticker data for the pair. See [ccxt documentation](https://github.com/ccxt/ccxt/wiki/Manual#price-tickers) for more details on the Ticker data structure. +- `runmode` - Property containing the current runmode. + +#### Example Usages: + +#### *available_pairs* + +``` python +if self.dp: + for pair, timeframe in self.dp.available_pairs: + print(f"available {pair}, {timeframe}") +``` + +#### *current_whitelist()* +Imagine you've developed a strategy that trades the `5m` timeframe using signals generated from a `1d` timeframe on the top 10 volume pairs by volume. + +The strategy might look something like this: + +*Scan through the top 10 pairs by volume using the `VolumePairList` every 5 minutes and use a 14 day ATR to buy and sell.* + +Due to the limited available data, it's very difficult to resample our `5m` candles into daily candles for use in a 14 day ATR. Most exchanges limit us to just 500 candles which effectively gives us around 1.74 daily candles. We need 14 days at least! + +Since we can't resample our data we will have to use an informative pair; and since our whitelist will be dynamic we don't know which pair(s) to use. + +This is where calling `self.dp.current_whitelist()` comes in handy. + +```python +class SampleStrategy(IStrategy): + # strategy init stuff... + + ticker_interval = '5m' + + # more strategy init stuff.. + + def informative_pairs(self): + + # get access to all pairs available in whitelist. + pairs = self.dp.current_whitelist() + # Assign tf to each pair so they can be downloaded and cached for strategy. + informative_pairs = [(pair, '1d') for pair in pairs] + return informative_pairs + + def populate_indicators(self, dataframe, metadata): + # Get the informative pair + informative = self.dp.get_pair_dataframe(pair=metadata['pair'], timeframe='1d') + # Get the 14 day ATR. + atr = ta.ATR(informative, timeperiod=14) + # Do other stuff +``` + +#### *get_pair_dataframe(pair, timeframe)* + +``` python +# fetch live / historical candle (OHLCV) data for the first informative pair +if self.dp: + inf_pair, inf_timeframe = self.informative_pairs()[0] + informative = self.dp.get_pair_dataframe(pair=inf_pair, + timeframe=inf_timeframe) +``` + +!!! Warning "Warning about backtesting" + Be carefull when using dataprovider in backtesting. `historic_ohlcv()` (and `get_pair_dataframe()` + for the backtesting runmode) provides the full time-range in one go, + so please be aware of it and make sure to not "look into the future" to avoid surprises when running in dry/live mode). + +!!! Warning "Warning in hyperopt" + This option cannot currently be used during hyperopt. + +#### *orderbook(pair, maximum)* + +``` python +if self.dp: + if self.dp.runmode.value in ('live', 'dry_run'): + ob = self.dp.orderbook(metadata['pair'], 1) + dataframe['best_bid'] = ob['bids'][0][0] + dataframe['best_ask'] = ob['asks'][0][0] +``` + +!!! Warning + The order book is not part of the historic data which means backtesting and hyperopt will not work if this + method is used. + +#### *ticker(pair)* + +``` python +if self.dp: + if self.dp.runmode.value in ('live', 'dry_run'): + ticker = self.dp.ticker(metadata['pair']) + dataframe['last_price'] = ticker['last'] + dataframe['volume24h'] = ticker['quoteVolume'] + dataframe['vwap'] = ticker['vwap'] +``` + +!!! Warning + Although the ticker data structure is a part of the ccxt Unified Interface, the values returned by this method can + vary for different exchanges. For instance, many exchanges do not return `vwap` values, the FTX exchange + does not always fills in the `last` field (so it can be None), etc. So you need to carefully verify the ticker + data returned from the exchange and add appropriate error handling / defaults. + +*** ### Additional data (Wallets) The strategy provides access to the `Wallets` object. This contains the current balances on the exchange. @@ -409,6 +492,7 @@ if self.wallets: - `get_used(asset)` - currently tied up balance (open orders) - `get_total(asset)` - total available balance - sum of the 2 above +*** ### Additional data (Trades) A history of Trades can be retrieved in the strategy by querying the database. @@ -422,7 +506,7 @@ from freqtrade.persistence import Trade The following example queries for the current pair and trades from today, however other filters can easily be added. ``` python -if self.config['runmode'] in ('live', 'dry_run'): +if self.config['runmode'].value in ('live', 'dry_run'): trades = Trade.get_trades([Trade.pair == metadata['pair'], Trade.open_date > datetime.utcnow() - timedelta(days=1), Trade.is_open == False, @@ -434,7 +518,7 @@ if self.config['runmode'] in ('live', 'dry_run'): Get amount of stake_currency currently invested in Trades: ``` python -if self.config['runmode'] in ('live', 'dry_run'): +if self.config['runmode'].value in ('live', 'dry_run'): total_stakes = Trade.total_open_trades_stakes() ``` @@ -442,7 +526,7 @@ Retrieve performance per pair. Returns a List of dicts per pair. ``` python -if self.config['runmode'] in ('live', 'dry_run'): +if self.config['runmode'].value in ('live', 'dry_run'): performance = Trade.get_overall_performance() ``` @@ -455,6 +539,51 @@ Sample return value: ETH/BTC had 5 trades, with a total profit of 1.5% (ratio of !!! Warning Trade history is not available during backtesting or hyperopt. +### Prevent trades from happening for a specific pair + +Freqtrade locks pairs automatically for the current candle (until that candle is over) when a pair is sold, preventing an immediate re-buy of that pair. + +Locked pairs will show the message `Pair is currently locked.`. + +#### Locking pairs from within the strategy + +Sometimes it may be desired to lock a pair after certain events happen (e.g. multiple losing trades in a row). + +Freqtrade has an easy method to do this from within the strategy, by calling `self.lock_pair(pair, until)`. +`until` must be a datetime object in the future, after which trading will be reenabled for that pair. + +Locks can also be lifted manually, by calling `self.unlock_pair(pair)`. + +To verify if a pair is currently locked, use `self.is_pair_locked(pair)`. + +!!! Note + Locked pairs are not persisted, so a restart of the bot, or calling `/reload_conf` will reset locked pairs. + +!!! Warning + Locking pairs is not functioning during backtesting. + +##### Pair locking example + +``` python +from freqtrade.persistence import Trade +from datetime import timedelta, datetime, timezone +# Put the above lines a the top of the strategy file, next to all the other imports +# -------- + +# Within populate indicators (or populate_buy): +if self.config['runmode'].value in ('live', 'dry_run'): + # fetch closed trades for the last 2 days + trades = Trade.get_trades([Trade.pair == metadata['pair'], + Trade.open_date > datetime.utcnow() - timedelta(days=2), + Trade.is_open == False, + ]).all() + # Analyze the conditions you'd like to lock the pair .... will probably be different for every strategy + sumprofit = sum(trade.close_profit for trade in trades) + if sumprofit < 0: + # Lock pair for 12 hours + self.lock_pair(metadata['pair'], until=datetime.now(timezone.utc) + timedelta(hours=12)) +``` + ### Print created dataframe To inspect the created dataframe, you can issue a print-statement in either `populate_buy_trend()` or `populate_sell_trend()`. @@ -479,11 +608,6 @@ def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: Printing more than a few rows is also possible (simply use `print(dataframe)` instead of `print(dataframe.tail())`), however not recommended, as that will be very verbose (~500 lines per pair every 5 seconds). -### Where can i find a strategy template? - -The strategy template is located in the file -[user_data/strategies/sample_strategy.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_strategy.py). - ### Specify custom strategy location If you want to use a strategy from a different directory you can pass `--strategy-path` @@ -492,6 +616,27 @@ If you want to use a strategy from a different directory you can pass `--strateg freqtrade trade --strategy AwesomeStrategy --strategy-path /some/directory ``` +### Derived strategies + +The strategies can be derived from other strategies. This avoids duplication of your custom strategy code. You can use this technique to override small parts of your main strategy, leaving the rest untouched: + +``` python +class MyAwesomeStrategy(IStrategy): + ... + stoploss = 0.13 + trailing_stop = False + # All other attributes and methods are here as they + # should be in any custom strategy... + ... + +class MyAwesomeStrategy2(MyAwesomeStrategy): + # Override something + stoploss = 0.08 + trailing_stop = True +``` + +Both attributes and methods may be overriden, altering behavior of the original strategy in a way you need. + ### Common mistakes when developing strategies Backtesting analyzes the whole time-range at once for performance reasons. Because of this, strategy authors need to make sure that strategies do not look-ahead into the future. diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 9e61bda65..d26d684ce 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -1,24 +1,28 @@ # Strategy analysis example -Debugging a strategy can be time-consuming. FreqTrade offers helper functions to visualize raw data. +Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data. +The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location. ## Setup ```python from pathlib import Path +from freqtrade.configuration import Configuration + # Customize these according to your needs. +# Initialize empty configuration object +config = Configuration.from_files([]) +# Optionally, use existing configuration file +# config = Configuration.from_files(["config.json"]) + # Define some constants -timeframe = "5m" +config["ticker_interval"] = "5m" # Name of the strategy class -strategy_name = 'SampleStrategy' -# Path to user data -user_data_dir = Path('user_data') -# Location of the strategy -strategy_location = user_data_dir / 'strategies' +config["strategy"] = "SampleStrategy" # Location of the data -data_location = Path(user_data_dir, 'data', 'binance') +data_location = Path(config['user_data_dir'], 'data', 'binance') # Pair to analyze - Only use one pair here pair = "BTC_USDT" ``` @@ -29,7 +33,7 @@ pair = "BTC_USDT" from freqtrade.data.history import load_pair_history candles = load_pair_history(datadir=data_location, - timeframe=timeframe, + timeframe=config["ticker_interval"], pair=pair) # Confirm success @@ -44,9 +48,7 @@ candles.head() ```python # Load strategy using values set above from freqtrade.resolvers import StrategyResolver -strategy = StrategyResolver({'strategy': strategy_name, - 'user_data_dir': user_data_dir, - 'strategy_path': strategy_location}).strategy +strategy = StrategyResolver.load_strategy(config) # Generate buy/sell signals using strategy df = strategy.analyze_ticker(candles, {'pair': pair}) @@ -86,7 +88,7 @@ Analyze a trades dataframe (also used below for plotting) from freqtrade.data.btanalysis import load_backtest_data # Load backtest results -trades = load_backtest_data(user_data_dir / "backtest_results/backtest-result.json") +trades = load_backtest_data(config["user_data_dir"] / "backtest_results/backtest-result.json") # Show value-counts per pair trades.groupby("pair")["sell_reason"].value_counts() @@ -119,7 +121,6 @@ from freqtrade.data.btanalysis import analyze_trade_parallelism # Analyze the above parallel_trades = analyze_trade_parallelism(trades, '5m') - parallel_trades.plot() ``` @@ -132,11 +133,14 @@ Freqtrade offers interactive plotting capabilities based on plotly. from freqtrade.plot.plotting import generate_candlestick_graph # Limit graph period to keep plotly quick and reactive +# Filter trades to one pair +trades_red = trades.loc[trades['pair'] == pair] + data_red = data['2019-06-01':'2019-06-10'] # Generate candlestick graph graph = generate_candlestick_graph(pair=pair, data=data_red, - trades=trades, + trades=trades_red, indicators1=['sma20', 'ema50', 'ema55'], indicators2=['rsi', 'macd', 'macdsignal', 'macdhist'] ) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index ed0c21a6e..f683ae8da 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -55,7 +55,7 @@ official commands. You can ask at any moment for help with `/help`. | `/reload_conf` | | Reloads the configuration file | `/show_config` | | Shows part of the current configuration with relevant settings to operation | `/status` | | Lists all open trades -| `/status table` | | List all open trades in a table format +| `/status table` | | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**) | `/count` | | Displays number of trades used and available | `/profit` | | Display a summary of your profit/loss from close trades and some stats about your performance | `/forcesell ` | | Instantly sells the given trade (Ignoring `minimum_roi`). diff --git a/docs/utils.md b/docs/utils.md index a9fbfc7d5..7ed31376f 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -36,6 +36,38 @@ optional arguments: └── sample_strategy.py ``` +## Create new config + +Creates a new configuration file, asking some questions which are important selections for a configuration. + +``` +usage: freqtrade new-config [-h] [-c PATH] + +optional arguments: + -h, --help show this help message and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. +``` + +!!! Warning + Only vital questions are asked. Freqtrade offers a lot more configuration possibilities, which are listed in the [Configuration documentation](configuration.md#configuration-parameters) + +### Create config examples + +``` +$ freqtrade new-config --config config_binance.json + +? Do you want to enable Dry-run (simulated trades)? Yes +? Please insert your stake currency: BTC +? Please insert your stake amount: 0.05 +? Please insert max_open_trades (Integer or 'unlimited'): 3 +? Please insert your timeframe (ticker interval): 5m +? Please insert your display Currency (for reporting): USD +? Select exchange binance +? Do you want to enable Telegram? No +``` + ## Create new strategy Creates a new strategy from a template similar to SampleStrategy. @@ -45,7 +77,7 @@ Results will be located in `user_data/strategies/.py`. ``` output usage: freqtrade new-strategy [-h] [--userdir PATH] [-s NAME] - [--template {full,minimal}] + [--template {full,minimal,advanced}] optional arguments: -h, --help show this help message and exit @@ -54,10 +86,10 @@ optional arguments: -s NAME, --strategy NAME Specify strategy class name which will be used by the bot. - --template {full,minimal} - Use a template which is either `minimal` or `full` - (containing multiple sample indicators). Default: - `full`. + --template {full,minimal,advanced} + Use a template which is either `minimal`, `full` + (containing multiple sample indicators) or `advanced`. + Default: `full`. ``` @@ -73,6 +105,12 @@ With custom user directory freqtrade new-strategy --userdir ~/.freqtrade/ --strategy AwesomeStrategy ``` +Using the advanced template (populates all optional functions and methods) + +```bash +freqtrade new-strategy --strategy AwesomeStrategy --template advanced +``` + ## Create new hyperopt Creates a new hyperopt from a template similar to SampleHyperopt. @@ -82,7 +120,7 @@ Results will be located in `user_data/hyperopts/.py`. ``` output usage: freqtrade new-hyperopt [-h] [--userdir PATH] [--hyperopt NAME] - [--template {full,minimal}] + [--template {full,minimal,advanced}] optional arguments: -h, --help show this help message and exit @@ -90,10 +128,10 @@ optional arguments: Path to userdata directory. --hyperopt NAME Specify hyperopt class name which will be used by the bot. - --template {full,minimal} - Use a template which is either `minimal` or `full` - (containing multiple sample indicators). Default: - `full`. + --template {full,minimal,advanced} + Use a template which is either `minimal`, `full` + (containing multiple sample indicators) or `advanced`. + Default: `full`. ``` ### Sample usage of new-hyperopt @@ -108,6 +146,97 @@ With custom user directory freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt ``` +## List Strategies and List Hyperopts + +Use the `list-strategies` subcommand to see all strategies in one particular directory and the `list-hyperopts` subcommand to list custom Hyperopts. + +These subcommands are useful for finding problems in your environment with loading strategies or hyperopt classes: modules with strategies or hyperopt classes that contain errors and failed to load are printed in red (LOAD FAILED), while strategies or hyperopt classes with duplicate names are printed in yellow (DUPLICATE NAME). + +``` +usage: freqtrade list-strategies [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [--strategy-path PATH] [-1] [--no-color] + +optional arguments: + -h, --help show this help message and exit + --strategy-path PATH Specify additional strategy lookup path. + -1, --one-column Print output in one column. + --no-color Disable colorization of hyperopt results. May be + useful if you are redirecting output to a file. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` +``` +usage: freqtrade list-hyperopts [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] + [--hyperopt-path PATH] [-1] [--no-color] + +optional arguments: + -h, --help show this help message and exit + --hyperopt-path PATH Specify additional lookup path for Hyperopt and + Hyperopt Loss functions. + -1, --one-column Print output in one column. + --no-color Disable colorization of hyperopt results. May be + useful if you are redirecting output to a file. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` + +!!! Warning + Using these commands will try to load all python files from a directory. This can be a security risk if untrusted files reside in this directory, since all module-level code is executed. + +Example: Search default strategies and hyperopts directories (within the default userdir). + +``` bash +freqtrade list-strategies +freqtrade list-hyperopts +``` + +Example: Search strategies and hyperopts directory within the userdir. + +``` bash +freqtrade list-strategies --userdir ~/.freqtrade/ +freqtrade list-hyperopts --userdir ~/.freqtrade/ +``` + +Example: Search dedicated strategy path. + +``` bash +freqtrade list-strategies --strategy-path ~/.freqtrade/strategies/ +``` + +Example: Search dedicated hyperopt path. + +``` bash +freqtrade list-hyperopt --hyperopt-path ~/.freqtrade/hyperopts/ +``` + ## List Exchanges Use the `list-exchanges` subcommand to see the exchanges available for the bot. @@ -135,23 +264,34 @@ All exchanges supported by the ccxt library: _1btcxe, acx, adara, allcoin, anxpr ## List Timeframes -Use the `list-timeframes` subcommand to see the list of ticker intervals (timeframes) available for the exchange. +Use the `list-timeframes` subcommand to see the list of timeframes (ticker intervals) available for the exchange. ``` -usage: freqtrade list-timeframes [-h] [--exchange EXCHANGE] [-1] +usage: freqtrade list-timeframes [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [-1] optional arguments: - -h, --help show this help message and exit - --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no - config is provided. - -1, --one-column Print output in one column. + -h, --help show this help message and exit + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no config is provided. + -1, --one-column Print output in one column. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: 'syslog', 'journald'. See the documentation for more details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). Multiple --config options may be used. Can be set to `-` + to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. ``` * Example: see the timeframes for the 'binance' exchange, set in the configuration file: ``` -$ freqtrade -c config_binance.json list-timeframes +$ freqtrade list-timeframes -c config_binance.json ... Timeframes available for the exchange `binance`: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4h, 6h, 8h, 12h, 1d, 3d, 1w, 1M ``` @@ -175,14 +315,16 @@ You can print info about any pair/market with these subcommands - and you can fi These subcommands have same usage and same set of available options: ``` -usage: freqtrade list-markets [-h] [--exchange EXCHANGE] [--print-list] - [--print-json] [-1] [--print-csv] +usage: freqtrade list-markets [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [--exchange EXCHANGE] + [--print-list] [--print-json] [-1] [--print-csv] [--base BASE_CURRENCY [BASE_CURRENCY ...]] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a] -usage: freqtrade list-pairs [-h] [--exchange EXCHANGE] [--print-list] - [--print-json] [-1] [--print-csv] +usage: freqtrade list-pairs [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [--exchange EXCHANGE] + [--print-list] [--print-json] [-1] [--print-csv] [--base BASE_CURRENCY [BASE_CURRENCY ...]] [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a] @@ -201,6 +343,22 @@ optional arguments: Specify quote currency(-ies). Space-separated list. -a, --all Print all pairs or market symbols. By default only active ones are shown. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: `config.json`). + Multiple --config options may be used. Can be set to + `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + ``` By default, only active pairs/markets are shown. Active pairs/markets are those that can currently be traded @@ -222,7 +380,7 @@ $ freqtrade list-pairs --quote USD --print-json human-readable list with summary: ``` -$ freqtrade -c config_binance.json list-pairs --all --base BTC ETH --quote USDT USD --print-list +$ freqtrade list-pairs -c config_binance.json --all --base BTC ETH --quote USDT USD --print-list ``` * Print all markets on exchange "Kraken", in the tabular format: @@ -270,17 +428,53 @@ You can list the hyperoptimization epochs the Hyperopt module evaluated previous ``` usage: freqtrade hyperopt-list [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--best] - [--profitable] [--no-color] [--print-json] - [--no-details] + [--profitable] [--min-trades INT] + [--max-trades INT] [--min-avg-time FLOAT] + [--max-avg-time FLOAT] [--min-avg-profit FLOAT] + [--max-avg-profit FLOAT] + [--min-total-profit FLOAT] + [--max-total-profit FLOAT] [--no-color] + [--print-json] [--no-details] + [--export-csv FILE] optional arguments: -h, --help show this help message and exit --best Select only best epochs. --profitable Select only profitable epochs. + --min-trades INT Select epochs with more than INT trades. + --max-trades INT Select epochs with less than INT trades. + --min-avg-time FLOAT Select epochs on above average time. + --max-avg-time FLOAT Select epochs on under average time. + --min-avg-profit FLOAT + Select epochs on above average profit. + --max-avg-profit FLOAT + Select epochs on below average profit. + --min-total-profit FLOAT + Select epochs on above total profit. + --max-total-profit FLOAT + Select epochs on below total profit. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. --print-json Print best result detailization in JSON format. --no-details Do not print best epoch details. + --export-csv FILE Export to CSV-File. This will disable table print. + Example: --export-csv hyperopt.csv + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. ``` ### Examples @@ -327,3 +521,48 @@ Prints JSON data with details for the last best epoch (i.e., the best of all epo ``` freqtrade hyperopt-show --best -n -1 --print-json --no-header ``` + +## Show trades + +Print selected (or all) trades from database to screen. + +``` +usage: freqtrade show-trades [-h] [-v] [--logfile FILE] [-V] [-c PATH] + [-d PATH] [--userdir PATH] [--db-url PATH] + [--trade-ids TRADE_IDS [TRADE_IDS ...]] + [--print-json] + +optional arguments: + -h, --help show this help message and exit + --db-url PATH Override trades database URL, this is useful in custom + deployments (default: `sqlite:///tradesv3.sqlite` for + Live Run mode, `sqlite:///tradesv3.dryrun.sqlite` for + Dry Run). + --trade-ids TRADE_IDS [TRADE_IDS ...] + Specify the list of trade ids. + --print-json Print output in JSON format. + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. +``` + +### Examples + +Print trades with id 2 and 3 as json + +``` bash +freqtrade show-trades --db-url sqlite:///tradesv3.sqlite --trade-ids 2 3 --print-json +``` diff --git a/docs/webhook-config.md b/docs/webhook-config.md index 9e0a34eae..70a41dd46 100644 --- a/docs/webhook-config.md +++ b/docs/webhook-config.md @@ -15,10 +15,20 @@ Sample configuration (tested using IFTTT). "value2": "limit {limit:8f}", "value3": "{stake_amount:8f} {stake_currency}" }, + "webhookbuycancel": { + "value1": "Cancelling Open Buy Order for {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, "webhooksell": { "value1": "Selling {pair}", "value2": "limit {limit:8f}", - "value3": "profit: {profit_amount:8f} {stake_currency}" + "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})" + }, + "webhooksellcancel": { + "value1": "Cancelling Open Sell Order for {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})" }, "webhookstatus": { "value1": "Status: {status}", @@ -40,10 +50,29 @@ Possible parameters are: * `exchange` * `pair` * `limit` +* `amount` +* `open_date` * `stake_amount` * `stake_currency` * `fiat_currency` * `order_type` +* `current_rate` + +### Webhookbuycancel + +The fields in `webhook.webhookbuycancel` are filled when the bot cancels a buy order. Parameters are filled using string.format. +Possible parameters are: + +* `exchange` +* `pair` +* `limit` +* `amount` +* `open_date` +* `stake_amount` +* `stake_currency` +* `fiat_currency` +* `order_type` +* `current_rate` ### Webhooksell @@ -58,7 +87,28 @@ Possible parameters are: * `open_rate` * `current_rate` * `profit_amount` -* `profit_percent` +* `profit_ratio` +* `stake_currency` +* `fiat_currency` +* `sell_reason` +* `order_type` +* `open_date` +* `close_date` + +### Webhooksellcancel + +The fields in `webhook.webhooksellcancel` are filled when the bot cancels a sell order. Parameters are filled using string.format. +Possible parameters are: + +* `exchange` +* `pair` +* `gain` +* `limit` +* `amount` +* `open_rate` +* `current_rate` +* `profit_amount` +* `profit_ratio` * `stake_currency` * `fiat_currency` * `sell_reason` diff --git a/environment.yml b/environment.yml index 4e8c1efcc..86ea03519 100644 --- a/environment.yml +++ b/environment.yml @@ -45,7 +45,7 @@ dependencies: - pip: # Required for app - cython - - coinmarketcap + - pycoingecko - ccxt - TA-Lib - py_find_1st diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 83fee0b0d..e96e7f530 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,44 +1,34 @@ -""" FreqTrade bot """ +""" Freqtrade bot """ __version__ = 'develop' if __version__ == 'develop': try: import subprocess + __version__ = 'develop-' + subprocess.check_output( ['git', 'log', '--format="%h"', '-n 1'], stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') + + # from datetime import datetime + # last_release = subprocess.check_output( + # ['git', 'tag'] + # ).decode('utf-8').split()[-1].split(".") + # # Releases are in the format "2020.1" - we increment the latest version for dev. + # prefix = f"{last_release[0]}.{int(last_release[1]) + 1}" + # dev_version = int(datetime.now().timestamp() // 1000) + # __version__ = f"{prefix}.dev{dev_version}" + + # subprocess.check_output( + # ['git', 'log', '--format="%h"', '-n 1'], + # stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') except Exception: # git not available, ignore - pass - - -class DependencyException(Exception): - """ - Indicates that an assumed dependency is not met. - This could happen when there is currently not enough money on the account. - """ - - -class OperationalException(Exception): - """ - Requires manual intervention and will usually stop the bot. - This happens when an exchange returns an unexpected error during runtime - or given configuration is invalid. - """ - - -class InvalidOrderException(Exception): - """ - This is returned when the order is not valid. Example: - If stoploss on exchange order is hit, then trying to cancel the order - should return this exception. - """ - - -class TemporaryError(Exception): - """ - Temporary network or exchange related error. - This could happen when an exchange is congested, unavailable, or the user - has networking problems. Usually resolves itself after a time. - """ + try: + # Try Fallback to freqtrade_commit file (created by CI while building docker image) + from pathlib import Path + versionfile = Path('./freqtrade_commit') + if versionfile.is_file(): + __version__ = f"docker-{versionfile.read_text()[:8]}" + except Exception: + pass diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py new file mode 100644 index 000000000..2d0c7733c --- /dev/null +++ b/freqtrade/commands/__init__.py @@ -0,0 +1,29 @@ +# flake8: noqa: F401 +""" +Commands module. +Contains all start-commands, subcommands and CLI Interface creation. + +Note: Be careful with file-scoped imports in these subfiles. + as they are parsed on startup, nothing containing optional modules should be loaded. +""" +from freqtrade.commands.arguments import Arguments +from freqtrade.commands.build_config_commands import start_new_config +from freqtrade.commands.data_commands import (start_convert_data, + start_download_data) +from freqtrade.commands.deploy_commands import (start_create_userdir, + start_new_hyperopt, + start_new_strategy) +from freqtrade.commands.hyperopt_commands import (start_hyperopt_list, + start_hyperopt_show) +from freqtrade.commands.list_commands import (start_list_exchanges, + start_list_hyperopts, + start_list_markets, + start_list_strategies, + start_list_timeframes, + start_show_trades) +from freqtrade.commands.optimize_commands import (start_backtesting, + start_edge, start_hyperopt) +from freqtrade.commands.pairlist_commands import start_test_pairlist +from freqtrade.commands.plot_commands import (start_plot_dataframe, + start_plot_profit) +from freqtrade.commands.trade_commands import start_trading diff --git a/freqtrade/configuration/arguments.py b/freqtrade/commands/arguments.py similarity index 67% rename from freqtrade/configuration/arguments.py rename to freqtrade/commands/arguments.py index 41c5c3957..a03da00ab 100644 --- a/freqtrade/configuration/arguments.py +++ b/freqtrade/commands/arguments.py @@ -6,8 +6,8 @@ from functools import partial from pathlib import Path from typing import Any, Dict, List, Optional -from freqtrade import constants -from freqtrade.configuration.cli_options import AVAILABLE_CLI_OPTIONS +from freqtrade.commands.cli_options import AVAILABLE_CLI_OPTIONS +from freqtrade.constants import DEFAULT_CONFIG ARGS_COMMON = ["verbosity", "logfile", "version", "config", "datadir", "user_data_dir"] @@ -30,6 +30,10 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] +ARGS_LIST_STRATEGIES = ["strategy_path", "print_one_column", "print_colorized"] + +ARGS_LIST_HYPEROPTS = ["hyperopt_path", "print_one_column", "print_colorized"] + ARGS_LIST_EXCHANGES = ["print_one_column", "list_exchanges_all"] ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"] @@ -41,28 +45,42 @@ ARGS_TEST_PAIRLIST = ["config", "quote_currencies", "print_one_column", "list_pa ARGS_CREATE_USERDIR = ["user_data_dir", "reset"] +ARGS_BUILD_CONFIG = ["config"] + ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] +ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] +ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] + ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange", - "timeframes", "erase"] + "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", - "timerange", "ticker_interval"] + "timerange", "ticker_interval", "no_trades"] ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "ticker_interval"] -ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "print_colorized", - "print_json", "hyperopt_list_no_details"] +ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"] + +ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", + "hyperopt_list_min_trades", "hyperopt_list_max_trades", + "hyperopt_list_min_avg_time", "hyperopt_list_max_avg_time", + "hyperopt_list_min_avg_profit", "hyperopt_list_max_avg_profit", + "hyperopt_list_min_total_profit", "hyperopt_list_max_total_profit", + "print_colorized", "print_json", "hyperopt_list_no_details", + "export_csv"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", "print_json", "hyperopt_show_no_header"] -NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs", - "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit"] +NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", + "list-markets", "list-pairs", "list-strategies", + "list-hyperopts", "hyperopt-list", "hyperopt-show", + "plot-dataframe", "plot-profit", "show-trades"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] @@ -96,10 +114,23 @@ class Arguments: # Workaround issue in argparse with action='append' and default value # (see https://bugs.python.org/issue16399) # Allow no-config for certain commands (like downloading / plotting) - if ('config' in parsed_arg and parsed_arg.config is None and - ((Path.cwd() / constants.DEFAULT_CONFIG).is_file() or - not ('command' in parsed_arg and parsed_arg.command in NO_CONF_REQURIED))): - parsed_arg.config = [constants.DEFAULT_CONFIG] + if ('config' in parsed_arg and parsed_arg.config is None): + conf_required = ('command' in parsed_arg and parsed_arg.command in NO_CONF_REQURIED) + + if 'user_data_dir' in parsed_arg and parsed_arg.user_data_dir is not None: + user_dir = parsed_arg.user_data_dir + else: + # Default case + user_dir = 'user_data' + # Try loading from "user_data/config.json" + cfgfile = Path(user_dir) / DEFAULT_CONFIG + if cfgfile.is_file(): + parsed_arg.config = [str(cfgfile)] + else: + # Else use "config.json". + cfgfile = Path.cwd() / DEFAULT_CONFIG + if cfgfile.is_file() or not conf_required: + parsed_arg.config = [DEFAULT_CONFIG] return parsed_arg @@ -127,13 +158,16 @@ class Arguments: self.parser = argparse.ArgumentParser(description='Free, open source crypto trading bot') self._build_args(optionlist=['version'], parser=self.parser) - from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge - from freqtrade.utils import (start_create_userdir, start_download_data, - start_hyperopt_list, start_hyperopt_show, - start_list_exchanges, start_list_markets, - start_new_hyperopt, start_new_strategy, - start_list_timeframes, start_test_pairlist, start_trading) - from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit + from freqtrade.commands import (start_create_userdir, start_convert_data, + start_download_data, + start_hyperopt_list, start_hyperopt_show, + start_list_exchanges, start_list_hyperopts, + start_list_markets, start_list_strategies, + start_list_timeframes, start_new_config, + start_new_hyperopt, start_new_strategy, + start_plot_dataframe, start_plot_profit, start_show_trades, + start_backtesting, start_hyperopt, start_edge, + start_test_pairlist, start_trading) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -173,6 +207,12 @@ class Arguments: create_userdir_cmd.set_defaults(func=start_create_userdir) self._build_args(optionlist=ARGS_CREATE_USERDIR, parser=create_userdir_cmd) + # add new-config subcommand + build_config_cmd = subparsers.add_parser('new-config', + help="Create new config") + build_config_cmd.set_defaults(func=start_new_config) + self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) + # add new-strategy subcommand build_strategy_cmd = subparsers.add_parser('new-strategy', help="Create new strategy") @@ -185,6 +225,24 @@ class Arguments: build_hyperopt_cmd.set_defaults(func=start_new_hyperopt) self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) + # Add list-strategies subcommand + list_strategies_cmd = subparsers.add_parser( + 'list-strategies', + help='Print available strategies.', + parents=[_common_parser], + ) + list_strategies_cmd.set_defaults(func=start_list_strategies) + self._build_args(optionlist=ARGS_LIST_STRATEGIES, parser=list_strategies_cmd) + + # Add list-hyperopts subcommand + list_hyperopts_cmd = subparsers.add_parser( + 'list-hyperopts', + help='Print available hyperopt classes.', + parents=[_common_parser], + ) + list_hyperopts_cmd.set_defaults(func=start_list_hyperopts) + self._build_args(optionlist=ARGS_LIST_HYPEROPTS, parser=list_hyperopts_cmd) + # Add list-exchanges subcommand list_exchanges_cmd = subparsers.add_parser( 'list-exchanges', @@ -238,6 +296,24 @@ class Arguments: download_data_cmd.set_defaults(func=start_download_data) self._build_args(optionlist=ARGS_DOWNLOAD_DATA, parser=download_data_cmd) + # Add convert-data subcommand + convert_data_cmd = subparsers.add_parser( + 'convert-data', + help='Convert candle (OHLCV) data from one format to another.', + parents=[_common_parser], + ) + convert_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=True)) + self._build_args(optionlist=ARGS_CONVERT_DATA_OHLCV, parser=convert_data_cmd) + + # Add convert-trade-data subcommand + convert_trade_data_cmd = subparsers.add_parser( + 'convert-trade-data', + help='Convert trade data from one format to another.', + parents=[_common_parser], + ) + convert_trade_data_cmd.set_defaults(func=partial(start_convert_data, ohlcv=False)) + self._build_args(optionlist=ARGS_CONVERT_DATA, parser=convert_trade_data_cmd) + # Add Plotting subcommand plot_dataframe_cmd = subparsers.add_parser( 'plot-dataframe', @@ -256,6 +332,15 @@ class Arguments: plot_profit_cmd.set_defaults(func=start_plot_profit) self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd) + # Add show-trades subcommand + show_trades = subparsers.add_parser( + 'show-trades', + help='Show trades.', + parents=[_common_parser], + ) + show_trades.set_defaults(func=start_show_trades) + self._build_args(optionlist=ARGS_SHOW_TRADES, parser=show_trades) + # Add hyperopt-list subcommand hyperopt_list_cmd = subparsers.add_parser( 'hyperopt-list', diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py new file mode 100644 index 000000000..87098f53c --- /dev/null +++ b/freqtrade/commands/build_config_commands.py @@ -0,0 +1,193 @@ +import logging +from pathlib import Path +from typing import Any, Dict + +from questionary import Separator, prompt + +from freqtrade.constants import UNLIMITED_STAKE_AMOUNT +from freqtrade.exchange import available_exchanges, MAP_EXCHANGE_CHILDCLASS +from freqtrade.misc import render_template +from freqtrade.exceptions import OperationalException +logger = logging.getLogger(__name__) + + +def validate_is_int(val): + try: + _ = int(val) + return True + except Exception: + return False + + +def validate_is_float(val): + try: + _ = float(val) + return True + except Exception: + return False + + +def ask_user_overwrite(config_path: Path) -> bool: + questions = [ + { + "type": "confirm", + "name": "overwrite", + "message": f"File {config_path} already exists. Overwrite?", + "default": False, + }, + ] + answers = prompt(questions) + return answers['overwrite'] + + +def ask_user_config() -> Dict[str, Any]: + """ + Ask user a few questions to build the configuration. + Interactive questions built using https://github.com/tmbo/questionary + :returns: Dict with keys to put into template + """ + questions = [ + { + "type": "confirm", + "name": "dry_run", + "message": "Do you want to enable Dry-run (simulated trades)?", + "default": True, + }, + { + "type": "text", + "name": "stake_currency", + "message": "Please insert your stake currency:", + "default": 'BTC', + }, + { + "type": "text", + "name": "stake_amount", + "message": "Please insert your stake amount:", + "default": "0.01", + "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), + }, + { + "type": "text", + "name": "max_open_trades", + "message": f"Please insert max_open_trades (Integer or '{UNLIMITED_STAKE_AMOUNT}'):", + "default": "3", + "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_int(val) + }, + { + "type": "text", + "name": "ticker_interval", + "message": "Please insert your timeframe (ticker interval):", + "default": "5m", + }, + { + "type": "text", + "name": "fiat_display_currency", + "message": "Please insert your display Currency (for reporting):", + "default": 'USD', + }, + { + "type": "select", + "name": "exchange_name", + "message": "Select exchange", + "choices": [ + "binance", + "binanceje", + "binanceus", + "bittrex", + "kraken", + Separator(), + "other", + ], + }, + { + "type": "autocomplete", + "name": "exchange_name", + "message": "Type your exchange name (Must be supported by ccxt)", + "choices": available_exchanges(), + "when": lambda x: x["exchange_name"] == 'other' + }, + { + "type": "password", + "name": "exchange_key", + "message": "Insert Exchange Key", + "when": lambda x: not x['dry_run'] + }, + { + "type": "password", + "name": "exchange_secret", + "message": "Insert Exchange Secret", + "when": lambda x: not x['dry_run'] + }, + { + "type": "confirm", + "name": "telegram", + "message": "Do you want to enable Telegram?", + "default": False, + }, + { + "type": "password", + "name": "telegram_token", + "message": "Insert Telegram token", + "when": lambda x: x['telegram'] + }, + { + "type": "text", + "name": "telegram_chat_id", + "message": "Insert Telegram chat id", + "when": lambda x: x['telegram'] + }, + ] + answers = prompt(questions) + + if not answers: + # Interrupted questionary sessions return an empty dict. + raise OperationalException("User interrupted interactive questions.") + + return answers + + +def deploy_new_config(config_path: Path, selections: Dict[str, Any]) -> None: + """ + Applies selections to the template and writes the result to config_path + :param config_path: Path object for new config file. Should not exist yet + :param selecions: Dict containing selections taken by the user. + """ + from jinja2.exceptions import TemplateNotFound + try: + exchange_template = MAP_EXCHANGE_CHILDCLASS.get( + selections['exchange_name'], selections['exchange_name']) + + selections['exchange'] = render_template( + templatefile=f"subtemplates/exchange_{exchange_template}.j2", + arguments=selections + ) + except TemplateNotFound: + selections['exchange'] = render_template( + templatefile="subtemplates/exchange_generic.j2", + arguments=selections + ) + + config_text = render_template(templatefile='base_config.json.j2', + arguments=selections) + + logger.info(f"Writing config to `{config_path}`.") + config_path.write_text(config_text) + + +def start_new_config(args: Dict[str, Any]) -> None: + """ + Create a new strategy from a template + Asking the user questions to fill out the templateaccordingly. + """ + + config_path = Path(args['config'][0]) + if config_path.exists(): + overwrite = ask_user_overwrite(config_path) + if overwrite: + config_path.unlink() + else: + raise OperationalException( + f"Configuration file `{config_path}` already exists. " + "Please delete it or use a different configuration file name.") + selections = ask_user_config() + deploy_new_config(config_path, selections) diff --git a/freqtrade/configuration/cli_options.py b/freqtrade/commands/cli_options.py similarity index 76% rename from freqtrade/configuration/cli_options.py rename to freqtrade/commands/cli_options.py index 30902dfe9..ee9208c33 100644 --- a/freqtrade/configuration/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -1,7 +1,7 @@ """ Definition of cli arguments used in arguments.py """ -import argparse +from argparse import ArgumentTypeError from freqtrade import __version__, constants @@ -12,7 +12,7 @@ def check_int_positive(value: str) -> int: if uint <= 0: raise ValueError except ValueError: - raise argparse.ArgumentTypeError( + raise ArgumentTypeError( f"{value} is invalid for this parameter, should be a positive integer value" ) return uint @@ -24,7 +24,7 @@ def check_int_nonzero(value: str) -> int: if uint == 0: raise ValueError except ValueError: - raise argparse.ArgumentTypeError( + raise ArgumentTypeError( f"{value} is invalid for this parameter, should be a non-zero integer value" ) return uint @@ -59,7 +59,8 @@ AVAILABLE_CLI_OPTIONS = { ), "config": Arg( '-c', '--config', - help=f'Specify configuration file (default: `{constants.DEFAULT_CONFIG}`). ' + help=f'Specify configuration file (default: `userdir/{constants.DEFAULT_CONFIG}` ' + f'or `config.json` whichever exists). ' f'Multiple --config options may be used. ' f'Can be set to `-` to read config from stdin.', action='append', @@ -118,14 +119,14 @@ AVAILABLE_CLI_OPTIONS = { help='Specify what timerange of data to use.', ), "max_open_trades": Arg( - '--max_open_trades', - help='Specify max_open_trades to use.', + '--max-open-trades', + help='Override the value of the `max_open_trades` configuration setting.', type=int, metavar='INT', ), "stake_amount": Arg( - '--stake_amount', - help='Specify stake_amount.', + '--stake-amount', + help='Override the value of the `stake_amount` configuration setting.', type=float, ), # Backtesting @@ -216,10 +217,17 @@ AVAILABLE_CLI_OPTIONS = { ), "print_json": Arg( '--print-json', - help='Print best result detailization in JSON format.', + help='Print output in JSON format.', action='store_true', default=False, ), + "export_csv": Arg( + '--export-csv', + help='Export to CSV-File.' + ' This will disable table print.' + ' Example: --export-csv hyperopt.csv', + metavar='FILE', + ), "hyperopt_jobs": Arg( '-j', '--job-workers', help='The number of concurrently running jobs for hyperoptimization ' @@ -256,7 +264,8 @@ AVAILABLE_CLI_OPTIONS = { help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). ' 'Different functions can generate completely different results, ' 'since the target for optimization is different. Built-in Hyperopt-loss-functions are: ' - 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss.' + 'DefaultHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, ' + 'SortinoHyperOptLoss, SortinoHyperOptLossDaily.' '(default: `%(default)s`).', metavar='NAME', default=constants.DEFAULT_HYPEROPT_LOSS, @@ -332,6 +341,30 @@ AVAILABLE_CLI_OPTIONS = { 'desired timeframe as specified as --timeframes/-t.', action='store_true', ), + "format_from": Arg( + '--format-from', + help='Source format for data conversion.', + choices=constants.AVAILABLE_DATAHANDLERS, + required=True, + ), + "format_to": Arg( + '--format-to', + help='Destination format for data conversion.', + choices=constants.AVAILABLE_DATAHANDLERS, + required=True, + ), + "dataformat_ohlcv": Arg( + '--data-format-ohlcv', + help='Storage format for downloaded candle (OHLCV) data. (default: `%(default)s`).', + choices=constants.AVAILABLE_DATAHANDLERS, + default='json' + ), + "dataformat_trades": Arg( + '--data-format-trades', + help='Storage format for downloaded trades data. (default: `%(default)s`).', + choices=constants.AVAILABLE_DATAHANDLERS, + default='jsongz' + ), "exchange": Arg( '--exchange', help=f'Exchange name (default: `{constants.DEFAULT_EXCHANGE}`). ' @@ -339,8 +372,8 @@ AVAILABLE_CLI_OPTIONS = { ), "timeframes": Arg( '-t', '--timeframes', - help=f'Specify which tickers to download. Space-separated list. ' - f'Default: `1m 5m`.', + help='Specify which tickers to download. Space-separated list. ' + 'Default: `1m 5m`.', choices=['1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w'], default=['1m', '5m'], @@ -354,24 +387,22 @@ AVAILABLE_CLI_OPTIONS = { # Templating options "template": Arg( '--template', - help='Use a template which is either `minimal` or ' - '`full` (containing multiple sample indicators). Default: `%(default)s`.', - choices=['full', 'minimal'], + help='Use a template which is either `minimal`, ' + '`full` (containing multiple sample indicators) or `advanced`. Default: `%(default)s`.', + choices=['full', 'minimal', 'advanced'], default='full', ), # Plot dataframe "indicators1": Arg( '--indicators1', help='Set indicators from your strategy you want in the first row of the graph. ' - 'Space-separated list. Example: `ema3 ema5`. Default: `%(default)s`.', - default=['sma', 'ema3', 'ema5'], + "Space-separated list. Example: `ema3 ema5`. Default: `['sma', 'ema3', 'ema5']`.", nargs='+', ), "indicators2": Arg( '--indicators2', help='Set indicators from your strategy you want in the third row of the graph. ' - 'Space-separated list. Example: `fastd fastk`. Default: `%(default)s`.', - default=['macd', 'macdsignal'], + "Space-separated list. Example: `fastd fastk`. Default: `['macd', 'macdsignal']`.", nargs='+', ), "plot_limit": Arg( @@ -382,6 +413,11 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=750, ), + "no_trades": Arg( + '--no-trades', + help='Skip using trades from backtesting file and DB.', + action='store_true', + ), "trade_source": Arg( '--trade-source', help='Specify the source for trades (Can be DB or file (backtest file)) ' @@ -389,6 +425,11 @@ AVAILABLE_CLI_OPTIONS = { choices=["DB", "file"], default="file", ), + "trade_ids": Arg( + '--trade-ids', + help='Specify the list of trade ids.', + nargs='+', + ), # hyperopt-list, hyperopt-show "hyperopt_list_profitable": Arg( '--profitable', @@ -400,6 +441,54 @@ AVAILABLE_CLI_OPTIONS = { help='Select only best epochs.', action='store_true', ), + "hyperopt_list_min_trades": Arg( + '--min-trades', + help='Select epochs with more than INT trades.', + type=check_int_positive, + metavar='INT', + ), + "hyperopt_list_max_trades": Arg( + '--max-trades', + help='Select epochs with less than INT trades.', + type=check_int_positive, + metavar='INT', + ), + "hyperopt_list_min_avg_time": Arg( + '--min-avg-time', + help='Select epochs on above average time.', + type=float, + metavar='FLOAT', + ), + "hyperopt_list_max_avg_time": Arg( + '--max-avg-time', + help='Select epochs on under average time.', + type=float, + metavar='FLOAT', + ), + "hyperopt_list_min_avg_profit": Arg( + '--min-avg-profit', + help='Select epochs on above average profit.', + type=float, + metavar='FLOAT', + ), + "hyperopt_list_max_avg_profit": Arg( + '--max-avg-profit', + help='Select epochs on below average profit.', + type=float, + metavar='FLOAT', + ), + "hyperopt_list_min_total_profit": Arg( + '--min-total-profit', + help='Select epochs on above total profit.', + type=float, + metavar='FLOAT', + ), + "hyperopt_list_max_total_profit": Arg( + '--max-total-profit', + help='Select epochs on below total profit.', + type=float, + metavar='FLOAT', + ), "hyperopt_list_no_details": Arg( '--no-details', help='Do not print best epoch details.', diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py new file mode 100644 index 000000000..fc3a49f1d --- /dev/null +++ b/freqtrade/commands/data_commands.py @@ -0,0 +1,90 @@ +import logging +import sys +from typing import Any, Dict, List + +import arrow + +from freqtrade.configuration import TimeRange, setup_utils_configuration +from freqtrade.data.converter import (convert_ohlcv_format, + convert_trades_format) +from freqtrade.data.history import (convert_trades_to_ohlcv, + refresh_backtest_ohlcv_data, + refresh_backtest_trades_data) +from freqtrade.exceptions import OperationalException +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_download_data(args: Dict[str, Any]) -> None: + """ + Download data (former download_backtest_data.py script) + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + timerange = TimeRange() + if 'days' in config: + time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") + timerange = TimeRange.parse_timerange(f'{time_since}-') + + if 'pairs' not in config: + raise OperationalException( + "Downloading data requires a list of pairs. " + "Please check the documentation on how to configure this.") + + logger.info(f'About to download pairs: {config["pairs"]}, ' + f'intervals: {config["timeframes"]} to {config["datadir"]}') + + pairs_not_available: List[str] = [] + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + # Manual validations of relevant settings + exchange.validate_pairs(config['pairs']) + for timeframe in config['timeframes']: + exchange.validate_timeframes(timeframe) + + try: + + if config.get('download_trades'): + pairs_not_available = refresh_backtest_trades_data( + exchange, pairs=config["pairs"], datadir=config['datadir'], + timerange=timerange, erase=bool(config.get("erase")), + data_format=config['dataformat_trades']) + + # Convert downloaded trade data to different timeframes + convert_trades_to_ohlcv( + pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=bool(config.get("erase")), + data_format_ohlcv=config['dataformat_ohlcv'], + data_format_trades=config['dataformat_trades'], + ) + else: + pairs_not_available = refresh_backtest_ohlcv_data( + exchange, pairs=config["pairs"], timeframes=config["timeframes"], + datadir=config['datadir'], timerange=timerange, erase=bool(config.get("erase")), + data_format=config['dataformat_ohlcv']) + + except KeyboardInterrupt: + sys.exit("SIGINT received, aborting ...") + + finally: + if pairs_not_available: + logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " + f"on exchange {exchange.name}.") + + +def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: + """ + Convert data from one format to another + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + if ohlcv: + convert_ohlcv_format(config, + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) + else: + convert_trades_format(config, + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase']) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py new file mode 100644 index 000000000..86562fa7c --- /dev/null +++ b/freqtrade/commands/deploy_commands.py @@ -0,0 +1,139 @@ +import logging +import sys +from pathlib import Path +from typing import Any, Dict + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.configuration.directory_operations import (copy_sample_files, + create_userdata_dir) +from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.exceptions import OperationalException +from freqtrade.misc import render_template, render_template_with_fallback +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_create_userdir(args: Dict[str, Any]) -> None: + """ + Create "user_data" directory to contain user data strategies, hyperopt, ...) + :param args: Cli args from Arguments() + :return: None + """ + if "user_data_dir" in args and args["user_data_dir"]: + userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) + copy_sample_files(userdir, overwrite=args["reset"]) + else: + logger.warning("`create-userdir` requires --userdir to be set.") + sys.exit(1) + + +def deploy_new_strategy(strategy_name: str, strategy_path: Path, subtemplate: str) -> None: + """ + Deploy new strategy from template to strategy_path + """ + fallback = 'full' + indicators = render_template_with_fallback( + templatefile=f"subtemplates/indicators_{subtemplate}.j2", + templatefallbackfile=f"subtemplates/indicators_{fallback}.j2", + ) + buy_trend = render_template_with_fallback( + templatefile=f"subtemplates/buy_trend_{subtemplate}.j2", + templatefallbackfile=f"subtemplates/buy_trend_{fallback}.j2", + ) + sell_trend = render_template_with_fallback( + templatefile=f"subtemplates/sell_trend_{subtemplate}.j2", + templatefallbackfile=f"subtemplates/sell_trend_{fallback}.j2", + ) + plot_config = render_template_with_fallback( + templatefile=f"subtemplates/plot_config_{subtemplate}.j2", + templatefallbackfile=f"subtemplates/plot_config_{fallback}.j2", + ) + additional_methods = render_template_with_fallback( + templatefile=f"subtemplates/strategy_methods_{subtemplate}.j2", + templatefallbackfile="subtemplates/strategy_methods_empty.j2", + ) + + strategy_text = render_template(templatefile='base_strategy.py.j2', + arguments={"strategy": strategy_name, + "indicators": indicators, + "buy_trend": buy_trend, + "sell_trend": sell_trend, + "plot_config": plot_config, + "additional_methods": additional_methods, + }) + + logger.info(f"Writing strategy to `{strategy_path}`.") + strategy_path.write_text(strategy_text) + + +def start_new_strategy(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if "strategy" in args and args["strategy"]: + if args["strategy"] == "DefaultStrategy": + raise OperationalException("DefaultStrategy is not allowed as name.") + + new_path = config['user_data_dir'] / USERPATH_STRATEGIES / (args["strategy"] + ".py") + + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Strategy Name.") + + deploy_new_strategy(args['strategy'], new_path, args['template']) + + else: + raise OperationalException("`new-strategy` requires --strategy to be set.") + + +def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: str) -> None: + """ + Deploys a new hyperopt template to hyperopt_path + """ + fallback = 'full' + buy_guards = render_template_with_fallback( + templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2", + templatefallbackfile=f"subtemplates/hyperopt_buy_guards_{fallback}.j2", + ) + sell_guards = render_template_with_fallback( + templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2", + templatefallbackfile=f"subtemplates/hyperopt_sell_guards_{fallback}.j2", + ) + buy_space = render_template_with_fallback( + templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2", + templatefallbackfile=f"subtemplates/hyperopt_buy_space_{fallback}.j2", + ) + sell_space = render_template_with_fallback( + templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2", + templatefallbackfile=f"subtemplates/hyperopt_sell_space_{fallback}.j2", + ) + + strategy_text = render_template(templatefile='base_hyperopt.py.j2', + arguments={"hyperopt": hyperopt_name, + "buy_guards": buy_guards, + "sell_guards": sell_guards, + "buy_space": buy_space, + "sell_space": sell_space, + }) + + logger.info(f"Writing hyperopt to `{hyperopt_path}`.") + hyperopt_path.write_text(strategy_text) + + +def start_new_hyperopt(args: Dict[str, Any]) -> None: + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if "hyperopt" in args and args["hyperopt"]: + if args["hyperopt"] == "DefaultHyperopt": + raise OperationalException("DefaultHyperopt is not allowed as name.") + + new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py") + + if new_path.exists(): + raise OperationalException(f"`{new_path}` already exists. " + "Please choose another Strategy Name.") + deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) + else: + raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py new file mode 100755 index 000000000..517f47d16 --- /dev/null +++ b/freqtrade/commands/hyperopt_commands.py @@ -0,0 +1,182 @@ +import logging +from operator import itemgetter +from typing import Any, Dict, List + +from colorama import init as colorama_init + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.exceptions import OperationalException +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_hyperopt_list(args: Dict[str, Any]) -> None: + """ + List hyperopt epochs previously evaluated + """ + from freqtrade.optimize.hyperopt import Hyperopt + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + print_colorized = config.get('print_colorized', False) + print_json = config.get('print_json', False) + export_csv = config.get('export_csv', None) + no_details = config.get('hyperopt_list_no_details', False) + no_header = False + + filteroptions = { + 'only_best': config.get('hyperopt_list_best', False), + 'only_profitable': config.get('hyperopt_list_profitable', False), + 'filter_min_trades': config.get('hyperopt_list_min_trades', 0), + 'filter_max_trades': config.get('hyperopt_list_max_trades', 0), + 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), + 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), + 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), + 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), + 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None) + } + + results_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + + # Previous evaluations + epochs = Hyperopt.load_previous_results(results_file) + total_epochs = len(epochs) + + epochs = _hyperopt_filter_epochs(epochs, filteroptions) + + if print_colorized: + colorama_init(autoreset=True) + + if not export_csv: + try: + print(Hyperopt.get_result_table(config, epochs, total_epochs, + not filteroptions['only_best'], print_colorized, 0)) + except KeyboardInterrupt: + print('User interrupted..') + + if epochs and not no_details: + sorted_epochs = sorted(epochs, key=itemgetter('loss')) + results = sorted_epochs[0] + Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) + + if epochs and export_csv: + Hyperopt.export_csv_file( + config, epochs, total_epochs, not filteroptions['only_best'], export_csv + ) + + +def start_hyperopt_show(args: Dict[str, Any]) -> None: + """ + Show details of a hyperopt epoch previously evaluated + """ + from freqtrade.optimize.hyperopt import Hyperopt + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + print_json = config.get('print_json', False) + no_header = config.get('hyperopt_show_no_header', False) + results_file = (config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + n = config.get('hyperopt_show_index', -1) + + filteroptions = { + 'only_best': config.get('hyperopt_list_best', False), + 'only_profitable': config.get('hyperopt_list_profitable', False), + 'filter_min_trades': config.get('hyperopt_list_min_trades', 0), + 'filter_max_trades': config.get('hyperopt_list_max_trades', 0), + 'filter_min_avg_time': config.get('hyperopt_list_min_avg_time', None), + 'filter_max_avg_time': config.get('hyperopt_list_max_avg_time', None), + 'filter_min_avg_profit': config.get('hyperopt_list_min_avg_profit', None), + 'filter_max_avg_profit': config.get('hyperopt_list_max_avg_profit', None), + 'filter_min_total_profit': config.get('hyperopt_list_min_total_profit', None), + 'filter_max_total_profit': config.get('hyperopt_list_max_total_profit', None) + } + + # Previous evaluations + epochs = Hyperopt.load_previous_results(results_file) + total_epochs = len(epochs) + + epochs = _hyperopt_filter_epochs(epochs, filteroptions) + filtered_epochs = len(epochs) + + if n > filtered_epochs: + raise OperationalException( + f"The index of the epoch to show should be less than {filtered_epochs + 1}.") + if n < -filtered_epochs: + raise OperationalException( + f"The index of the epoch to show should be greater than {-filtered_epochs - 1}.") + + # Translate epoch index from human-readable format to pythonic + if n > 0: + n -= 1 + + if epochs: + val = epochs[n] + Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, + header_str="Epoch details") + + +def _hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: + """ + Filter our items from the list of hyperopt results + """ + if filteroptions['only_best']: + epochs = [x for x in epochs if x['is_best']] + if filteroptions['only_profitable']: + epochs = [x for x in epochs if x['results_metrics']['profit'] > 0] + if filteroptions['filter_min_trades'] > 0: + epochs = [ + x for x in epochs + if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades'] + ] + if filteroptions['filter_max_trades'] > 0: + epochs = [ + x for x in epochs + if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades'] + ] + if filteroptions['filter_min_avg_time'] is not None: + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs + if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time'] + ] + if filteroptions['filter_max_avg_time'] is not None: + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs + if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time'] + ] + if filteroptions['filter_min_avg_profit'] is not None: + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs + if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit'] + ] + if filteroptions['filter_max_avg_profit'] is not None: + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs + if x['results_metrics']['avg_profit'] < filteroptions['filter_max_avg_profit'] + ] + if filteroptions['filter_min_total_profit'] is not None: + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs + if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit'] + ] + if filteroptions['filter_max_total_profit'] is not None: + epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0] + epochs = [ + x for x in epochs + if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit'] + ] + + logger.info(f"{len(epochs)} " + + ("best " if filteroptions['only_best'] else "") + + ("profitable " if filteroptions['only_profitable'] else "") + + "epochs found.") + + return epochs diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py new file mode 100644 index 000000000..e5131f9b2 --- /dev/null +++ b/freqtrade/commands/list_commands.py @@ -0,0 +1,226 @@ +import csv +import logging +import sys +from collections import OrderedDict +from pathlib import Path +from typing import Any, Dict, List + +from colorama import init as colorama_init +from colorama import Fore, Style +import rapidjson +from tabulate import tabulate + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import (available_exchanges, ccxt_exchanges, + market_is_active, symbol_is_pair) +from freqtrade.misc import plural +from freqtrade.resolvers import ExchangeResolver, StrategyResolver +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_list_exchanges(args: Dict[str, Any]) -> None: + """ + Print available exchanges + :param args: Cli args from Arguments() + :return: None + """ + exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges() + if args['print_one_column']: + print('\n'.join(exchanges)) + else: + if args['list_exchanges_all']: + print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}") + else: + print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") + + +def _print_objs_tabular(objs: List, print_colorized: bool) -> None: + if print_colorized: + colorama_init(autoreset=True) + red = Fore.RED + yellow = Fore.YELLOW + reset = Style.RESET_ALL + else: + red = '' + yellow = '' + reset = '' + + names = [s['name'] for s in objs] + objss_to_print = [{ + 'name': s['name'] if s['name'] else "--", + 'location': s['location'].name, + 'status': (red + "LOAD FAILED" + reset if s['class'] is None + else "OK" if names.count(s['name']) == 1 + else yellow + "DUPLICATE NAME" + reset) + } for s in objs] + + print(tabulate(objss_to_print, headers='keys', tablefmt='psql', stralign='right')) + + +def start_list_strategies(args: Dict[str, Any]) -> None: + """ + Print files with Strategy custom classes available in the directory + """ + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) + strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column']) + # Sort alphabetically + strategy_objs = sorted(strategy_objs, key=lambda x: x['name']) + + if args['print_one_column']: + print('\n'.join([s['name'] for s in strategy_objs])) + else: + _print_objs_tabular(strategy_objs, config.get('print_colorized', False)) + + +def start_list_hyperopts(args: Dict[str, Any]) -> None: + """ + Print files with HyperOpt custom classes available in the directory + """ + from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver + + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + directory = Path(config.get('hyperopt_path', config['user_data_dir'] / USERPATH_HYPEROPTS)) + hyperopt_objs = HyperOptResolver.search_all_objects(directory, not args['print_one_column']) + # Sort alphabetically + hyperopt_objs = sorted(hyperopt_objs, key=lambda x: x['name']) + + if args['print_one_column']: + print('\n'.join([s['name'] for s in hyperopt_objs])) + else: + _print_objs_tabular(hyperopt_objs, config.get('print_colorized', False)) + + +def start_list_timeframes(args: Dict[str, Any]) -> None: + """ + Print ticker intervals (timeframes) available on Exchange + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + # Do not use ticker_interval set in the config + config['ticker_interval'] = None + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + if args['print_one_column']: + print('\n'.join(exchange.timeframes)) + else: + print(f"Timeframes available for the exchange `{exchange.name}`: " + f"{', '.join(exchange.timeframes)}") + + +def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: + """ + Print pairs/markets on the exchange + :param args: Cli args from Arguments() + :param pairs_only: if True print only pairs, otherwise print all instruments (markets) + :return: None + """ + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + # Init exchange + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + # By default only active pairs/markets are to be shown + active_only = not args.get('list_pairs_all', False) + + base_currencies = args.get('base_currencies', []) + quote_currencies = args.get('quote_currencies', []) + + try: + pairs = exchange.get_markets(base_currencies=base_currencies, + quote_currencies=quote_currencies, + pairs_only=pairs_only, + active_only=active_only) + # Sort the pairs/markets by symbol + pairs = OrderedDict(sorted(pairs.items())) + except Exception as e: + raise OperationalException(f"Cannot get markets. Reason: {e}") from e + + else: + summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") + + ("active " if active_only else "") + + (plural(len(pairs), "pair" if pairs_only else "market")) + + (f" with {', '.join(base_currencies)} as base " + f"{plural(len(base_currencies), 'currency', 'currencies')}" + if base_currencies else "") + + (" and" if base_currencies and quote_currencies else "") + + (f" with {', '.join(quote_currencies)} as quote " + f"{plural(len(quote_currencies), 'currency', 'currencies')}" + if quote_currencies else "")) + + headers = ["Id", "Symbol", "Base", "Quote", "Active", + *(['Is pair'] if not pairs_only else [])] + + tabular_data = [] + for _, v in pairs.items(): + tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], + 'Base': v['base'], 'Quote': v['quote'], + 'Active': market_is_active(v), + **({'Is pair': symbol_is_pair(v['symbol'])} + if not pairs_only else {})}) + + if (args.get('print_one_column', False) or + args.get('list_pairs_print_json', False) or + args.get('print_csv', False)): + # Print summary string in the log in case of machine-readable + # regular formats. + logger.info(f"{summary_str}.") + else: + # Print empty string separating leading logs and output in case of + # human-readable formats. + print() + + if len(pairs): + if args.get('print_list', False): + # print data as a list, with human-readable summary + print(f"{summary_str}: {', '.join(pairs.keys())}.") + elif args.get('print_one_column', False): + print('\n'.join(pairs.keys())) + elif args.get('list_pairs_print_json', False): + print(rapidjson.dumps(list(pairs.keys()), default=str)) + elif args.get('print_csv', False): + writer = csv.DictWriter(sys.stdout, fieldnames=headers) + writer.writeheader() + writer.writerows(tabular_data) + else: + # print data as a table, with the human-readable summary + print(f"{summary_str}:") + print(tabulate(tabular_data, headers='keys', tablefmt='psql', stralign='right')) + elif not (args.get('print_one_column', False) or + args.get('list_pairs_print_json', False) or + args.get('print_csv', False)): + print(f"{summary_str}.") + + +def start_show_trades(args: Dict[str, Any]) -> None: + """ + Show trades + """ + from freqtrade.persistence import init, Trade + import json + config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) + + if 'db_url' not in config: + raise OperationalException("--db-url is required for this command.") + + logger.info(f'Using DB: "{config["db_url"]}"') + init(config['db_url'], clean_open_orders=False) + tfilter = [] + + if config.get('trade_ids'): + tfilter.append(Trade.id.in_(config['trade_ids'])) + + trades = Trade.get_trades(tfilter).all() + logger.info(f"Printing {len(trades)} Trades: ") + if config.get('print_json', False): + print(json.dumps([trade.to_json() for trade in trades], indent=4)) + else: + for trade in trades: + print(trade) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py new file mode 100644 index 000000000..2fc605926 --- /dev/null +++ b/freqtrade/commands/optimize_commands.py @@ -0,0 +1,107 @@ +import logging +from typing import Any, Dict + +from freqtrade import constants +from freqtrade.configuration import setup_utils_configuration +from freqtrade.exceptions import DependencyException, OperationalException +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def setup_optimize_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: + """ + Prepare the configuration for the Hyperopt module + :param args: Cli args from Arguments() + :return: Configuration + """ + config = setup_utils_configuration(args, method) + + no_unlimited_runmodes = { + RunMode.BACKTEST: 'backtesting', + RunMode.HYPEROPT: 'hyperoptimization', + } + if (method in no_unlimited_runmodes.keys() and + config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT): + raise DependencyException( + f'The value of `stake_amount` cannot be set as "{constants.UNLIMITED_STAKE_AMOUNT}" ' + f'for {no_unlimited_runmodes[method]}') + + return config + + +def start_backtesting(args: Dict[str, Any]) -> None: + """ + Start Backtesting script + :param args: Cli args from Arguments() + :return: None + """ + # Import here to avoid loading backtesting module when it's not used + from freqtrade.optimize.backtesting import Backtesting + + # Initialize configuration + config = setup_optimize_configuration(args, RunMode.BACKTEST) + + logger.info('Starting freqtrade in Backtesting mode') + + # Initialize backtesting object + backtesting = Backtesting(config) + backtesting.start() + + +def start_hyperopt(args: Dict[str, Any]) -> None: + """ + Start hyperopt script + :param args: Cli args from Arguments() + :return: None + """ + # Import here to avoid loading hyperopt module when it's not used + try: + from filelock import FileLock, Timeout + from freqtrade.optimize.hyperopt import Hyperopt + except ImportError as e: + raise OperationalException( + f"{e}. Please ensure that the hyperopt dependencies are installed.") from e + # Initialize configuration + config = setup_optimize_configuration(args, RunMode.HYPEROPT) + + logger.info('Starting freqtrade in Hyperopt mode') + + lock = FileLock(Hyperopt.get_lock_filename(config)) + + try: + with lock.acquire(timeout=1): + + # Remove noisy log messages + logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) + logging.getLogger('filelock').setLevel(logging.WARNING) + + # Initialize backtesting object + hyperopt = Hyperopt(config) + hyperopt.start() + + except Timeout: + logger.info("Another running instance of freqtrade Hyperopt detected.") + logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. " + "Hyperopt module is resource hungry. Please run your Hyperopt sequentially " + "or on separate machines.") + logger.info("Quitting now.") + # TODO: return False here in order to help freqtrade to exit + # with non-zero exit code... + # Same in Edge and Backtesting start() functions. + + +def start_edge(args: Dict[str, Any]) -> None: + """ + Start Edge script + :param args: Cli args from Arguments() + :return: None + """ + from freqtrade.optimize.edge_cli import EdgeCli + # Initialize configuration + config = setup_optimize_configuration(args, RunMode.EDGE) + logger.info('Starting freqtrade in Edge mode') + + # Initialize Edge object + edge_cli = EdgeCli(config) + edge_cli.start() diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py new file mode 100644 index 000000000..bf0b217a5 --- /dev/null +++ b/freqtrade/commands/pairlist_commands.py @@ -0,0 +1,42 @@ +import logging +from typing import Any, Dict + +import rapidjson + +from freqtrade.configuration import setup_utils_configuration +from freqtrade.resolvers import ExchangeResolver +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def start_test_pairlist(args: Dict[str, Any]) -> None: + """ + Test Pairlist configuration + """ + from freqtrade.pairlist.pairlistmanager import PairListManager + config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) + + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) + + quote_currencies = args.get('quote_currencies') + if not quote_currencies: + quote_currencies = [config.get('stake_currency')] + results = {} + for curr in quote_currencies: + config['stake_currency'] = curr + # Do not use ticker_interval set in the config + pairlists = PairListManager(exchange, config) + pairlists.refresh_pairlist() + results[curr] = pairlists.whitelist + + for curr, pairlist in results.items(): + if not args.get('print_one_column', False): + print(f"Pairs for {curr}: ") + + if args.get('print_one_column', False): + print('\n'.join(pairlist)) + elif args.get('list_pairs_print_json', False): + print(rapidjson.dumps(list(pairlist), default=str)) + else: + print(pairlist) diff --git a/freqtrade/plot/plot_utils.py b/freqtrade/commands/plot_commands.py similarity index 85% rename from freqtrade/plot/plot_utils.py rename to freqtrade/commands/plot_commands.py index 8de0eb9e7..5e547acb0 100644 --- a/freqtrade/plot/plot_utils.py +++ b/freqtrade/commands/plot_commands.py @@ -1,11 +1,11 @@ from typing import Any, Dict -from freqtrade import OperationalException +from freqtrade.configuration import setup_utils_configuration +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.utils import setup_utils_configuration -def validate_plot_args(args: Dict[str, Any]): +def validate_plot_args(args: Dict[str, Any]) -> None: if not args.get('datadir') and not args.get('config'): raise OperationalException( "You need to specify either `--datadir` or `--config` " diff --git a/freqtrade/commands/trade_commands.py b/freqtrade/commands/trade_commands.py new file mode 100644 index 000000000..c058e4f9d --- /dev/null +++ b/freqtrade/commands/trade_commands.py @@ -0,0 +1,30 @@ +import logging + +from typing import Any, Dict + + +logger = logging.getLogger(__name__) + + +def start_trading(args: Dict[str, Any]) -> int: + """ + Main entry point for trading mode + """ + # Import here to avoid loading worker module when it's not used + from freqtrade.worker import Worker + + # Create and run worker + worker = None + try: + worker = Worker(args) + worker.run() + except Exception as e: + logger.error(str(e)) + logger.exception("Fatal exception!") + except KeyboardInterrupt: + logger.info('SIGINT received, aborting ...') + finally: + if worker: + logger.info("worker found ... calling exit") + worker.exit() + return 0 diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 63c38d8c5..d41ac97ec 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,5 +1,7 @@ -from freqtrade.configuration.arguments import Arguments # noqa: F401 -from freqtrade.configuration.check_exchange import check_exchange, remove_credentials # noqa: F401 -from freqtrade.configuration.timerange import TimeRange # noqa: F401 -from freqtrade.configuration.configuration import Configuration # noqa: F401 -from freqtrade.configuration.config_validation import validate_config_consistency # noqa: F401 +# flake8: noqa: F401 + +from freqtrade.configuration.config_setup import setup_utils_configuration +from freqtrade.configuration.check_exchange import check_exchange, remove_credentials +from freqtrade.configuration.timerange import TimeRange +from freqtrade.configuration.configuration import Configuration +from freqtrade.configuration.config_validation import validate_config_consistency diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index c739de692..92daaf251 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -1,16 +1,16 @@ import logging from typing import Any, Dict -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, - is_exchange_known_ccxt, is_exchange_bad, + is_exchange_bad, is_exchange_known_ccxt, is_exchange_officially_supported) from freqtrade.state import RunMode logger = logging.getLogger(__name__) -def remove_credentials(config: Dict[str, Any]): +def remove_credentials(config: Dict[str, Any]) -> None: """ Removes exchange keys from the configuration and specifies dry-run Used for backtesting / hyperopt / edge and utils. diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py new file mode 100644 index 000000000..64f283e42 --- /dev/null +++ b/freqtrade/configuration/config_setup.py @@ -0,0 +1,25 @@ +import logging +from typing import Any, Dict + +from .config_validation import validate_config_consistency +from .configuration import Configuration +from .check_exchange import remove_credentials +from freqtrade.state import RunMode + +logger = logging.getLogger(__name__) + + +def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: + """ + Prepare the configuration for utils subcommands + :param args: Cli args from Arguments() + :return: Configuration + """ + configuration = Configuration(args, method) + config = configuration.get_config() + + # Ensure we do not use Exchange credentials + remove_credentials(config) + validate_config_consistency(config) + + return config diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 068364884..5ba7ff294 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -1,10 +1,12 @@ import logging +from copy import deepcopy from typing import Any, Dict from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match -from freqtrade import constants, OperationalException +from freqtrade import constants +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode logger = logging.getLogger(__name__) @@ -41,15 +43,20 @@ def validate_config_schema(conf: Dict[str, Any]) -> Dict[str, Any]: :param conf: Config in JSON format :return: Returns the config if valid, otherwise throw an exception """ + conf_schema = deepcopy(constants.CONF_SCHEMA) + if conf.get('runmode', RunMode.OTHER) in (RunMode.DRY_RUN, RunMode.LIVE): + conf_schema['required'] = constants.SCHEMA_TRADE_REQUIRED + else: + conf_schema['required'] = constants.SCHEMA_MINIMAL_REQUIRED try: - FreqtradeValidator(constants.CONF_SCHEMA).validate(conf) + FreqtradeValidator(conf_schema).validate(conf) return conf except ValidationError as e: logger.critical( f"Invalid configuration. See config.json.example. Reason: {e}" ) raise ValidationError( - best_match(Draft4Validator(constants.CONF_SCHEMA).iter_errors(conf)).message + best_match(Draft4Validator(conf_schema).iter_errors(conf)).message ) @@ -66,12 +73,24 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: _validate_trailing_stoploss(conf) _validate_edge(conf) _validate_whitelist(conf) + _validate_unlimited_amount(conf) # validate configuration before returning logger.info('Validating configuration ...') validate_config_schema(conf) +def _validate_unlimited_amount(conf: Dict[str, Any]) -> None: + """ + If edge is disabled, either max_open_trades or stake_amount need to be set. + :raise: OperationalException if config validation failed + """ + if (not conf.get('edge', {}).get('enabled') + and conf.get('max_open_trades') == float('inf') + and conf.get('stake_amount') == constants.UNLIMITED_STAKE_AMOUNT): + raise OperationalException("`max_open_trades` and `stake_amount` cannot both be unlimited.") + + def _validate_trailing_stoploss(conf: Dict[str, Any]) -> None: if conf.get('stoploss') == 0.0: @@ -131,15 +150,3 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: if (pl.get('method') == 'StaticPairList' and not conf.get('exchange', {}).get('pair_whitelist')): raise OperationalException("StaticPairList requires pair_whitelist to be set.") - - if pl.get('method') == 'StaticPairList': - stake = conf['stake_currency'] - invalid_pairs = [] - for pair in conf['exchange'].get('pair_whitelist'): - if not pair.endswith(f'/{stake}'): - invalid_pairs.append(pair) - - if invalid_pairs: - raise OperationalException( - f"Stake-currency '{stake}' not compatible with pair-whitelist. " - f"Please remove the following pairs: {invalid_pairs}") diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index e517a0558..7edd9bca1 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -7,15 +7,16 @@ from copy import deepcopy from pathlib import Path from typing import Any, Callable, Dict, List, Optional -from freqtrade import OperationalException, constants +from freqtrade import constants from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings from freqtrade.configuration.directory_operations import (create_datadir, create_userdata_dir) from freqtrade.configuration.load_config import load_config_file +from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts, json_load -from freqtrade.state import RunMode, TRADING_MODES, NON_UTIL_MODES +from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode logger = logging.getLogger(__name__) @@ -95,6 +96,8 @@ class Configuration: # Keep a copy of the original configuration file config['original_config'] = deepcopy(config) + self._process_logging_options(config) + self._process_runmode(config) self._process_common_options(config) @@ -145,8 +148,6 @@ class Configuration: def _process_common_options(self, config: Dict[str, Any]) -> None: - self._process_logging_options(config) - # Set strategy if not specified in config and or if it's non default if self.args.get("strategy") or not config.get('strategy'): config.update({'strategy': self.args.get("strategy")}) @@ -166,10 +167,6 @@ class Configuration: if 'sd_notify' in self.args and self.args["sd_notify"]: config['internals'].update({'sd_notify': True}) - self._args_to_config(config, argname='dry_run', - logstring='Parameter --dry-run detected, ' - 'overriding dry_run to: {} ...') - def _process_datadir_options(self, config: Dict[str, Any]) -> None: """ Extract information for sys.argv and load directory configurations @@ -199,6 +196,7 @@ class Configuration: if self.args.get('exportfilename'): self._args_to_config(config, argname='exportfilename', logstring='Storing backtest results to {} ...') + config['exportfilename'] = Path(config['exportfilename']) else: config['exportfilename'] = (config['user_data_dir'] / 'backtest_results/backtest-result.json') @@ -223,13 +221,13 @@ class Configuration: logger.info('max_open_trades set to unlimited ...') elif 'max_open_trades' in self.args and self.args["max_open_trades"]: config.update({'max_open_trades': self.args["max_open_trades"]}) - logger.info('Parameter --max_open_trades detected, ' + logger.info('Parameter --max-open-trades detected, ' 'overriding max_open_trades to: %s ...', config.get('max_open_trades')) elif config['runmode'] in NON_UTIL_MODES: logger.info('Using max_open_trades: %s ...', config.get('max_open_trades')) self._args_to_config(config, argname='stake_amount', - logstring='Parameter --stake_amount detected, ' + logstring='Parameter --stake-amount detected, ' 'overriding stake_amount to: {} ...') self._args_to_config(config, argname='fee', @@ -285,6 +283,9 @@ class Configuration: self._args_to_config(config, argname='print_json', logstring='Parameter --print-json detected ...') + self._args_to_config(config, argname='export_csv', + logstring='Parameter --export-csv detected: {}') + self._args_to_config(config, argname='hyperopt_jobs', logstring='Parameter -j/--job-workers detected: {}') @@ -309,6 +310,30 @@ class Configuration: self._args_to_config(config, argname='hyperopt_list_profitable', logstring='Parameter --profitable detected: {}') + self._args_to_config(config, argname='hyperopt_list_min_trades', + logstring='Parameter --min-trades detected: {}') + + self._args_to_config(config, argname='hyperopt_list_max_trades', + logstring='Parameter --max-trades detected: {}') + + self._args_to_config(config, argname='hyperopt_list_min_avg_time', + logstring='Parameter --min-avg-time detected: {}') + + self._args_to_config(config, argname='hyperopt_list_max_avg_time', + logstring='Parameter --max-avg-time detected: {}') + + self._args_to_config(config, argname='hyperopt_list_min_avg_profit', + logstring='Parameter --min-avg-profit detected: {}') + + self._args_to_config(config, argname='hyperopt_list_max_avg_profit', + logstring='Parameter --max-avg-profit detected: {}') + + self._args_to_config(config, argname='hyperopt_list_min_total_profit', + logstring='Parameter --min-total-profit detected: {}') + + self._args_to_config(config, argname='hyperopt_list_max_total_profit', + logstring='Parameter --max-total-profit detected: {}') + self._args_to_config(config, argname='hyperopt_list_no_details', logstring='Parameter --no-details detected: {}') @@ -326,28 +351,46 @@ class Configuration: self._args_to_config(config, argname='indicators2', logstring='Using indicators2: {}') + self._args_to_config(config, argname='trade_ids', + logstring='Filtering on trade_ids: {}') + self._args_to_config(config, argname='plot_limit', logstring='Limiting plot to: {}') + self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') self._args_to_config(config, argname='erase', logstring='Erase detected. Deleting existing data.') + self._args_to_config(config, argname='no_trades', + logstring='Parameter --no-trades detected.') + self._args_to_config(config, argname='timeframes', logstring='timeframes --timeframes: {}') self._args_to_config(config, argname='days', logstring='Detected --days: {}') + self._args_to_config(config, argname='download_trades', logstring='Detected --dl-trades: {}') + self._args_to_config(config, argname='dataformat_ohlcv', + logstring='Using "{}" to store OHLCV data.') + + self._args_to_config(config, argname='dataformat_trades', + logstring='Using "{}" to store trades data.') + def _process_runmode(self, config: Dict[str, Any]) -> None: + self._args_to_config(config, argname='dry_run', + logstring='Parameter --dry-run detected, ' + 'overriding dry_run to: {} ...') + if not self.runmode: # Handle real mode, infer dry/live from config self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE - logger.info(f"Runmode set to {self.runmode}.") + logger.info(f"Runmode set to {self.runmode.value}.") config.update({'runmode': self.runmode}) @@ -403,7 +446,7 @@ class Configuration: config['pairs'] = config.get('exchange', {}).get('pair_whitelist') else: # Fall back to /dl_path/pairs.json - pairs_file = Path(config['datadir']) / "pairs.json" + pairs_file = config['datadir'] / "pairs.json" if pairs_file.exists(): with pairs_file.open('r') as f: config['pairs'] = json_load(f) diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index b1e3535a3..3999ea422 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -5,7 +5,7 @@ Functions to handle deprecated settings import logging from typing import Any, Dict -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) def check_conflicting_settings(config: Dict[str, Any], section1: str, name1: str, - section2: str, name2: str): + section2: str, name2: str) -> None: section1_config = config.get(section1, {}) section2_config = config.get(section2, {}) if name1 in section1_config and name2 in section2_config: @@ -28,7 +28,7 @@ def check_conflicting_settings(config: Dict[str, Any], def process_deprecated_setting(config: Dict[str, Any], section1: str, name1: str, - section2: str, name2: str): + section2: str, name2: str) -> None: section2_config = config.get(section2, {}) if name2 in section2_config: @@ -58,25 +58,12 @@ def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal', 'experimental', 'ignore_roi_if_buy_signal') - if not config.get('pairlists') and not config.get('pairlists'): - config['pairlists'] = [{'method': 'StaticPairList'}] + if (config.get('edge', {}).get('enabled', False) + and 'capital_available_percentage' in config.get('edge', {})): logger.warning( "DEPRECATED: " - "Pairlists must be defined explicitly in the future." - "Defaulting to StaticPairList for now.") - - if config.get('pairlist', {}).get("method") == 'VolumePairList': - logger.warning( - "DEPRECATED: " - f"Using VolumePairList in pairlist is deprecated and must be moved to pairlists. " - "Please refer to the docs on configuration details") - pl = {'method': 'VolumePairList'} - pl.update(config.get('pairlist', {}).get('config')) - config['pairlists'].append(pl) - - if config.get('pairlist', {}).get('config', {}).get('precision_filter'): - logger.warning( - "DEPRECATED: " - f"Using precision_filter setting is deprecated and has been replaced by" - "PrecisionFilter. Please refer to the docs on configuration details") - config['pairlists'].append({'method': 'PrecisionFilter'}) + "Using 'edge.capital_available_percentage' has been deprecated in favor of " + "'tradable_balance_ratio'. Please migrate your configuration to " + "'tradable_balance_ratio' and remove 'capital_available_percentage' " + "from the edge configuration." + ) diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index 3dd76a025..6b8c8cb5a 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -3,13 +3,13 @@ import shutil from pathlib import Path from typing import Any, Dict, Optional -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.constants import USER_DATA_FILES logger = logging.getLogger(__name__) -def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str: +def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Path: folder = Path(datadir) if datadir else Path(f"{config['user_data_dir']}/data") if not datadir: @@ -20,10 +20,10 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> str if not folder.is_dir(): folder.mkdir(parents=True) logger.info(f'Created data directory: {datadir}') - return str(folder) + return folder -def create_userdata_dir(directory: str, create_dir=False) -> Path: +def create_userdata_dir(directory: str, create_dir: bool = False) -> Path: """ Create userdata directory structure. if create_dir is True, then the parent-directory will be created if it does not exist. @@ -33,8 +33,8 @@ def create_userdata_dir(directory: str, create_dir=False) -> Path: :param create_dir: Create directory if it does not exist. :return: Path object containing the directory """ - sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "notebooks", - "plot", "strategies", ] + sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "logs", + "notebooks", "plot", "strategies", ] folder = Path(directory) if not folder.is_dir(): if create_dir: diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 7a3ca1798..a24ee3d0a 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -1,13 +1,15 @@ """ This module contain functions to load the configuration file """ -import rapidjson import logging +import re import sys +from pathlib import Path from typing import Any, Dict -from freqtrade import OperationalException +import rapidjson +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) @@ -15,6 +17,26 @@ logger = logging.getLogger(__name__) CONFIG_PARSE_MODE = rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS +def log_config_error_range(path: str, errmsg: str) -> str: + """ + Parses configuration file and prints range around error + """ + if path != '-': + offsetlist = re.findall(r'(?<=Parse\serror\sat\soffset\s)\d+', errmsg) + if offsetlist: + offset = int(offsetlist[0]) + text = Path(path).read_text() + # Fetch an offset of 80 characters around the error line + subtext = text[offset-min(80, offset):offset+80] + segments = subtext.split('\n') + if len(segments) > 3: + # Remove first and last lines, to avoid odd truncations + return '\n'.join(segments[1:-1]) + else: + return subtext + return '' + + def load_config_file(path: str) -> Dict[str, Any]: """ Loads a config file from the given path @@ -29,5 +51,12 @@ def load_config_file(path: str) -> Dict[str, Any]: raise OperationalException( f'Config file "{path}" not found!' ' Please create a config file or check whether it exists.') + except rapidjson.JSONDecodeError as e: + err_range = log_config_error_range(path, str(e)) + raise OperationalException( + f'{e}\n' + f'Please verify the following segment of your configuration:\n{err_range}' + if err_range else 'Please verify your configuration file for syntax errors.' + ) return config diff --git a/freqtrade/configuration/timerange.py b/freqtrade/configuration/timerange.py index a8be873df..151003999 100644 --- a/freqtrade/configuration/timerange.py +++ b/freqtrade/configuration/timerange.py @@ -7,6 +7,7 @@ from typing import Optional import arrow + logger = logging.getLogger(__name__) @@ -30,7 +31,7 @@ class TimeRange: return (self.starttype == other.starttype and self.stoptype == other.stoptype and self.startts == other.startts and self.stopts == other.stopts) - def subtract_start(self, seconds) -> None: + def subtract_start(self, seconds: int) -> None: """ Subtracts from startts if startts is set. :param seconds: Seconds to subtract from starttime @@ -44,7 +45,7 @@ class TimeRange: """ Adjust startts by candles. Applies only if no startup-candles have been available. - :param timeframe_secs: Ticker timeframe in seconds e.g. `timeframe_to_seconds('5m')` + :param timeframe_secs: Timeframe in seconds e.g. `timeframe_to_seconds('5m')` :param startup_candles: Number of candles to move start-date forward :param min_date: Minimum data date loaded. Key kriterium to decide if start-time has to be moved @@ -59,7 +60,7 @@ class TimeRange: self.starttype = 'date' @staticmethod - def parse_timerange(text: Optional[str]): + def parse_timerange(text: Optional[str]) -> 'TimeRange': """ Parse the value of the argument --timerange to determine what is the range desired :param text: value from --timerange diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 5c7190b41..e56586bbc 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -10,41 +10,43 @@ HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec DEFAULT_HYPEROPT_LOSS = 'DefaultHyperOptLoss' DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' -DEFAULT_DB_DRYRUN_URL = 'sqlite://' +DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' DEFAULT_AMOUNT_RESERVE_PERCENT = 0.05 REQUIRED_ORDERTIF = ['buy', 'sell'] REQUIRED_ORDERTYPES = ['buy', 'sell', 'stoploss', 'stoploss_on_exchange'] +ORDERBOOK_SIDES = ['ask', 'bid'] ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] -AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'PrecisionFilter', 'PriceFilter'] +AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', + 'PrecisionFilter', 'PriceFilter', 'SpreadFilter'] +AVAILABLE_DATAHANDLERS = ['json', 'jsongz'] DRY_RUN_WALLET = 1000 MATH_CLOSE_PREC = 1e-14 # Precision used for float comparisons +DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume'] +# Don't modify sequence of DEFAULT_TRADES_COLUMNS +# it has wide consequences for stored trades files +DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost'] USERPATH_HYPEROPTS = 'hyperopts' -USERPATH_STRATEGY = 'strategies' +USERPATH_STRATEGIES = 'strategies' +USERPATH_NOTEBOOKS = 'notebooks' # Soure files with destination directories within user-directory USER_DATA_FILES = { - 'sample_strategy.py': USERPATH_STRATEGY, + 'sample_strategy.py': USERPATH_STRATEGIES, 'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS, 'sample_hyperopt_loss.py': USERPATH_HYPEROPTS, 'sample_hyperopt.py': USERPATH_HYPEROPTS, - 'strategy_analysis_example.ipynb': 'notebooks', + 'strategy_analysis_example.ipynb': USERPATH_NOTEBOOKS, } -TIMEFRAMES = [ - '1m', '3m', '5m', '15m', '30m', - '1h', '2h', '4h', '6h', '8h', '12h', - '1d', '3d', '1w', -] - SUPPORTED_FIAT = [ "AUD", "BRL", "CAD", "CHF", "CLP", "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HUF", "IDR", "ILS", "INR", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD", "PHP", "PKR", "PLN", "RUB", "SEK", "SGD", "THB", "TRY", "TWD", "ZAR", "USD", - "BTC", "XBT", "ETH", "XRP", "LTC", "BCH", "USDT" + "BTC", "ETH", "XRP", "LTC", "BCH" ] MINIMAL_CONFIG = { @@ -66,16 +68,27 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1}, - 'ticker_interval': {'type': 'string', 'enum': TIMEFRAMES}, - 'stake_currency': {'type': 'string', 'enum': ['BTC', 'XBT', 'ETH', 'USDT', 'EUR', 'USD']}, + 'ticker_interval': {'type': 'string'}, + 'stake_currency': {'type': 'string'}, 'stake_amount': { 'type': ['number', 'string'], 'minimum': 0.0001, 'pattern': UNLIMITED_STAKE_AMOUNT }, + 'tradable_balance_ratio': { + 'type': 'number', + 'minimum': 0.1, + 'maximum': 1, + 'default': 0.99 + }, + 'amend_last_stake_amount': {'type': 'boolean', 'default': False}, + 'last_stake_amount_min_ratio': { + 'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5 + }, 'fiat_display_currency': {'type': 'string', 'enum': SUPPORTED_FIAT}, 'dry_run': {'type': 'boolean'}, 'dry_run_wallet': {'type': 'number', 'default': DRY_RUN_WALLET}, + 'cancel_open_orders_on_exit': {'type': 'boolean', 'default': False}, 'process_only_new_candles': {'type': 'boolean'}, 'minimal_roi': { 'type': 'object', @@ -105,15 +118,16 @@ CONF_SCHEMA = { 'minimum': 0, 'maximum': 1, 'exclusiveMaximum': False, - 'use_order_book': {'type': 'boolean'}, - 'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1}, - 'check_depth_of_market': { - 'type': 'object', - 'properties': { - 'enabled': {'type': 'boolean'}, - 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, - } - }, + }, + 'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'bid'}, + 'use_order_book': {'type': 'boolean'}, + 'order_book_top': {'type': 'integer', 'maximum': 20, 'minimum': 1}, + 'check_depth_of_market': { + 'type': 'object', + 'properties': { + 'enabled': {'type': 'boolean'}, + 'bids_to_ask_delta': {'type': 'number', 'minimum': 0}, + } }, }, 'required': ['ask_last_balance'] @@ -121,6 +135,7 @@ CONF_SCHEMA = { 'ask_strategy': { 'type': 'object', 'properties': { + 'price_side': {'type': 'string', 'enum': ORDERBOOK_SIDES, 'default': 'ask'}, 'use_order_book': {'type': 'boolean'}, 'order_book_min': {'type': 'integer', 'minimum': 1}, 'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50}, @@ -185,7 +200,9 @@ CONF_SCHEMA = { 'properties': { 'enabled': {'type': 'boolean'}, 'webhookbuy': {'type': 'object'}, + 'webhookbuycancel': {'type': 'object'}, 'webhooksell': {'type': 'object'}, + 'webhooksellcancel': {'type': 'object'}, 'webhookstatus': {'type': 'object'}, }, }, @@ -209,11 +226,22 @@ CONF_SCHEMA = { 'forcebuy_enable': {'type': 'boolean'}, 'internals': { 'type': 'object', + 'default': {}, 'properties': { 'process_throttle_secs': {'type': 'integer'}, 'interval': {'type': 'integer'}, 'sd_notify': {'type': 'boolean'}, } + }, + 'dataformat_ohlcv': { + 'type': 'string', + 'enum': AVAILABLE_DATAHANDLERS, + 'default': 'json' + }, + 'dataformat_trades': { + 'type': 'string', + 'enum': AVAILABLE_DATAHANDLERS, + 'default': 'jsongz' } }, 'definitions': { @@ -230,7 +258,6 @@ CONF_SCHEMA = { 'type': 'array', 'items': { 'type': 'string', - 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' }, 'uniqueItems': True }, @@ -238,7 +265,6 @@ CONF_SCHEMA = { 'type': 'array', 'items': { 'type': 'string', - 'pattern': '^[0-9A-Z]+/[0-9A-Z]+$' }, 'uniqueItems': True }, @@ -266,19 +292,40 @@ CONF_SCHEMA = { 'max_trade_duration_minute': {'type': 'integer'}, 'remove_pumps': {'type': 'boolean'} }, - 'required': ['process_throttle_secs', 'allowed_risk', 'capital_available_percentage'] + 'required': ['process_throttle_secs', 'allowed_risk'] } }, - 'required': [ - 'exchange', - 'max_open_trades', - 'stake_currency', - 'stake_amount', - 'dry_run', - 'dry_run_wallet', - 'bid_strategy', - 'unfilledtimeout', - 'stoploss', - 'minimal_roi', - ] +} + +SCHEMA_TRADE_REQUIRED = [ + 'exchange', + 'max_open_trades', + 'stake_currency', + 'stake_amount', + 'tradable_balance_ratio', + 'last_stake_amount_min_ratio', + 'dry_run', + 'dry_run_wallet', + 'ask_strategy', + 'bid_strategy', + 'unfilledtimeout', + 'stoploss', + 'minimal_roi', + 'internals', + 'dataformat_ohlcv', + 'dataformat_trades', +] + +SCHEMA_MINIMAL_REQUIRED = [ + 'exchange', + 'dry_run', + 'dataformat_ohlcv', + 'dataformat_trades', +] + +CANCEL_REASON = { + "TIMEOUT": "cancelled due to timeout", + "PARTIALLY_FILLED": "partially filled - keeping order open", + "ALL_CANCELLED": "cancelled (all unfilled and partially filled open orders cancelled)", + "CANCELLED_ON_EXCHANGE": "cancelled on exchange", } diff --git a/freqtrade/data/btanalysis.py b/freqtrade/data/btanalysis.py index 2fc931a9b..b0c642c1d 100644 --- a/freqtrade/data/btanalysis.py +++ b/freqtrade/data/btanalysis.py @@ -3,7 +3,7 @@ Helpers when analyzing backtest data """ import logging from pathlib import Path -from typing import Dict +from typing import Dict, Union, Tuple import numpy as np import pandas as pd @@ -20,7 +20,7 @@ BT_DATA_COLUMNS = ["pair", "profitperc", "open_time", "close_time", "index", "du "open_rate", "close_rate", "open_at_end", "sell_reason"] -def load_backtest_data(filename) -> pd.DataFrame: +def load_backtest_data(filename: Union[Path, str]) -> pd.DataFrame: """ Load backtest data file. :param filename: pathlib.Path object, or string pointing to the file. @@ -47,7 +47,7 @@ def load_backtest_data(filename) -> pd.DataFrame: utc=True, infer_datetime_format=True ) - df['profitabs'] = df['close_rate'] - df['open_rate'] + df['profit'] = df['close_rate'] - df['open_rate'] df = df.sort_values("open_time").reset_index(drop=True) return df @@ -111,7 +111,7 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: t.calc_profit(), t.calc_profit_ratio(), t.open_rate, t.close_rate, t.amount, (round((t.close_date.timestamp() - t.open_date.timestamp()) / 60, 2) - if t.close_date else None), + if t.close_date else None), t.sell_reason, t.fee_open, t.fee_close, t.open_rate_requested, @@ -129,38 +129,56 @@ def load_trades_from_db(db_url: str) -> pd.DataFrame: return trades -def load_trades(source: str, db_url: str, exportfilename: str) -> pd.DataFrame: +def load_trades(source: str, db_url: str, exportfilename: Path, + no_trades: bool = False) -> pd.DataFrame: """ Based on configuration option "trade_source": * loads data from DB (using `db_url`) * loads data from backtestfile (using `exportfilename`) + :param source: "DB" or "file" - specify source to load from + :param db_url: sqlalchemy formatted url to a database + :param exportfilename: Json file generated by backtesting + :param no_trades: Skip using trades, only return backtesting data columns + :return: DataFrame containing trades """ + if no_trades: + df = pd.DataFrame(columns=BT_DATA_COLUMNS) + return df + if source == "DB": return load_trades_from_db(db_url) elif source == "file": - return load_backtest_data(Path(exportfilename)) + return load_backtest_data(exportfilename) -def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame) -> pd.DataFrame: +def extract_trades_of_period(dataframe: pd.DataFrame, trades: pd.DataFrame, + date_index=False) -> pd.DataFrame: """ Compare trades and backtested pair DataFrames to get trades performed on backtested period :return: the DataFrame of a trades of period """ - trades = trades.loc[(trades['open_time'] >= dataframe.iloc[0]['date']) & - (trades['close_time'] <= dataframe.iloc[-1]['date'])] + if date_index: + trades_start = dataframe.index[0] + trades_stop = dataframe.index[-1] + else: + trades_start = dataframe.iloc[0]['date'] + trades_stop = dataframe.iloc[-1]['date'] + trades = trades.loc[(trades['open_time'] >= trades_start) & + (trades['close_time'] <= trades_stop)] return trades -def combine_tickers_with_mean(tickers: Dict[str, pd.DataFrame], column: str = "close"): +def combine_dataframes_with_mean(data: Dict[str, pd.DataFrame], + column: str = "close") -> pd.DataFrame: """ Combine multiple dataframes "column" - :param tickers: Dict of Dataframes, dict key should be pair. + :param data: Dict of Dataframes, dict key should be pair. :param column: Column in the original dataframes to use :return: DataFrame with the column renamed to the dict key, and a column named mean, containing the mean of all pairs. """ - df_comb = pd.concat([tickers[pair].set_index('date').rename( - {column: pair}, axis=1)[pair] for pair in tickers], axis=1) + df_comb = pd.concat([data[pair].set_index('date').rename( + {column: pair}, axis=1)[pair] for pair in data], axis=1) df_comb['mean'] = df_comb.mean(axis=1) @@ -187,3 +205,30 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str, # FFill to get continuous df[col_name] = df[col_name].ffill() return df + + +def calculate_max_drawdown(trades: pd.DataFrame, *, date_col: str = 'close_time', + value_col: str = 'profitperc' + ) -> Tuple[float, pd.Timestamp, pd.Timestamp]: + """ + Calculate max drawdown and the corresponding close dates + :param trades: DataFrame containing trades (requires columns close_time and profitperc) + :param date_col: Column in DataFrame to use for dates (defaults to 'close_time') + :param value_col: Column in DataFrame to use for values (defaults to 'profitperc') + :return: Tuple (float, highdate, lowdate) with absolute max drawdown, high and low time + :raise: ValueError if trade-dataframe was found empty. + """ + if len(trades) == 0: + raise ValueError("Trade dataframe empty.") + profit_results = trades.sort_values(date_col).reset_index(drop=True) + max_drawdown_df = pd.DataFrame() + max_drawdown_df['cumulative'] = profit_results[value_col].cumsum() + max_drawdown_df['high_value'] = max_drawdown_df['cumulative'].cummax() + max_drawdown_df['drawdown'] = max_drawdown_df['cumulative'] - max_drawdown_df['high_value'] + + idxmin = max_drawdown_df['drawdown'].idxmin() + if idxmin == 0: + raise ValueError("No losing trade, therefore no drawdown.") + high_date = profit_results.loc[max_drawdown_df.iloc[:idxmin]['high_value'].idxmax(), date_col] + low_date = profit_results.loc[idxmin, date_col] + return abs(min(max_drawdown_df['drawdown'])), high_date, low_date diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index e45dd451e..0ef7955a4 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -1,21 +1,27 @@ """ Functions to convert data from one format to another """ +import itertools import logging +from datetime import datetime, timezone +from operator import itemgetter +from typing import Any, Dict, List import pandas as pd from pandas import DataFrame, to_datetime +from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, + DEFAULT_TRADES_COLUMNS) logger = logging.getLogger(__name__) -def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *, - fill_missing: bool = True, - drop_incomplete: bool = True) -> DataFrame: +def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *, + fill_missing: bool = True, drop_incomplete: bool = True) -> DataFrame: """ - Converts a ticker-list (format ccxt.fetch_ohlcv) to a Dataframe - :param ticker: ticker list, as returned by exchange.async_get_candle_history + Converts a list with candle (OHLCV) data (in format returned by ccxt.fetch_ohlcv) + to a Dataframe + :param ohlcv: list with candle (OHLCV) data, as returned by exchange.async_get_candle_history :param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data :param pair: Pair this data is for (used to warn if fillup was necessary) :param fill_missing: fill up missing candles with 0 candles @@ -23,23 +29,40 @@ def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *, :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete :return: DataFrame """ - logger.debug("Parsing tickerlist to dataframe") - cols = ['date', 'open', 'high', 'low', 'close', 'volume'] - frame = DataFrame(ticker, columns=cols) + logger.debug(f"Converting candle (OHLCV) data to dataframe for pair {pair}.") + cols = DEFAULT_DATAFRAME_COLUMNS + df = DataFrame(ohlcv, columns=cols) - frame['date'] = to_datetime(frame['date'], - unit='ms', - utc=True, - infer_datetime_format=True) + df['date'] = to_datetime(df['date'], unit='ms', utc=True, infer_datetime_format=True) - # Some exchanges return int values for volume and even for ohlc. + # Some exchanges return int values for Volume and even for OHLC. # Convert them since TA-LIB indicators used in the strategy assume floats # and fail with exception... - frame = frame.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float', - 'volume': 'float'}) + df = df.astype(dtype={'open': 'float', 'high': 'float', 'low': 'float', 'close': 'float', + 'volume': 'float'}) + return clean_ohlcv_dataframe(df, timeframe, pair, + fill_missing=fill_missing, + drop_incomplete=drop_incomplete) + +def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *, + fill_missing: bool = True, + drop_incomplete: bool = True) -> DataFrame: + """ + Clense a OHLCV dataframe by + * Grouping it by date (removes duplicate tics) + * dropping last candles if requested + * Filling up missing data (if requested) + :param data: DataFrame containing candle (OHLCV) data. + :param timeframe: timeframe (e.g. 5m). Used to fill up eventual missing data + :param pair: Pair this data is for (used to warn if fillup was necessary) + :param fill_missing: fill up missing candles with 0 candles + (see ohlcv_fill_up_missing_data for details) + :param drop_incomplete: Drop the last candle of the dataframe, assuming it's incomplete + :return: DataFrame + """ # group by index and aggregate results to eliminate duplicate ticks - frame = frame.groupby(by='date', as_index=False, sort=True).agg({ + data = data.groupby(by='date', as_index=False, sort=True).agg({ 'open': 'first', 'high': 'max', 'low': 'min', @@ -48,13 +71,13 @@ def parse_ticker_dataframe(ticker: list, timeframe: str, pair: str, *, }) # eliminate partial candle if drop_incomplete: - frame.drop(frame.tail(1).index, inplace=True) + data.drop(data.tail(1).index, inplace=True) logger.debug('Dropping last candle') if fill_missing: - return ohlcv_fill_up_missing_data(frame, timeframe, pair) + return ohlcv_fill_up_missing_data(data, timeframe, pair) else: - return frame + return data def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) -> DataFrame: @@ -65,16 +88,16 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) """ from freqtrade.exchange import timeframe_to_minutes - ohlc_dict = { + ohlcv_dict = { 'open': 'first', 'high': 'max', 'low': 'min', 'close': 'last', 'volume': 'sum' } - ticker_minutes = timeframe_to_minutes(timeframe) + timeframe_minutes = timeframe_to_minutes(timeframe) # Resample to create "NAN" values - df = dataframe.resample(f'{ticker_minutes}min', on='date').agg(ohlc_dict) + df = dataframe.resample(f'{timeframe_minutes}min', on='date').agg(ohlcv_dict) # Forwardfill close for missing columns df['close'] = df['close'].fillna(method='ffill') @@ -92,8 +115,26 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) return df +def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date') -> DataFrame: + """ + Trim dataframe based on given timerange + :param df: Dataframe to trim + :param timerange: timerange (use start and end date if available) + :param: df_date_col: Column in the dataframe to use as Date column + :return: trimmed dataframe + """ + if timerange.starttype == 'date': + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + df = df.loc[df[df_date_col] >= start, :] + if timerange.stoptype == 'date': + stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) + df = df.loc[df[df_date_col] <= stop, :] + return df + + def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: """ + TODO: This should get a dedicated test Gets order book list, returns dataframe with below format per suggested by creslin ------------------------------------------------------------------- b_sum b_size bids asks a_size a_sum @@ -116,23 +157,105 @@ def order_book_to_dataframe(bids: list, asks: list) -> DataFrame: return frame -def trades_to_ohlcv(trades: list, timeframe: str) -> list: +def trades_remove_duplicates(trades: List[List]) -> List[List]: """ - Converts trades list to ohlcv list + Removes duplicates from the trades list. + Uses itertools.groupby to avoid converting to pandas. + Tests show it as being pretty efficient on lists of 4M Lists. + :param trades: List of Lists with constants.DEFAULT_TRADES_COLUMNS as columns + :return: same format as above, but with duplicates removed + """ + return [i for i, _ in itertools.groupby(sorted(trades, key=itemgetter(0)))] + + +def trades_dict_to_list(trades: List[Dict]) -> List[List]: + """ + Convert fetch_trades result into a List (to be more memory efficient). :param trades: List of trades, as returned by ccxt.fetch_trades. - :param timeframe: Ticker timeframe to resample data to - :return: ohlcv timeframe as list (as returned by ccxt.fetch_ohlcv) + :return: List of Lists, with constants.DEFAULT_TRADES_COLUMNS as columns + """ + return [[t[col] for col in DEFAULT_TRADES_COLUMNS] for t in trades] + + +def trades_to_ohlcv(trades: List, timeframe: str) -> DataFrame: + """ + Converts trades list to OHLCV list + TODO: This should get a dedicated test + :param trades: List of trades, as returned by ccxt.fetch_trades. + :param timeframe: Timeframe to resample data to + :return: OHLCV Dataframe. """ from freqtrade.exchange import timeframe_to_minutes - ticker_minutes = timeframe_to_minutes(timeframe) - df = pd.DataFrame(trades) - df['datetime'] = pd.to_datetime(df['datetime']) - df = df.set_index('datetime') + timeframe_minutes = timeframe_to_minutes(timeframe) + df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS) + df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', + utc=True,) + df = df.set_index('timestamp') - df_new = df['price'].resample(f'{ticker_minutes}min').ohlc() - df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum() - df_new['date'] = df_new.index.astype("int64") // 10 ** 6 + df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc() + df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum() + df_new['date'] = df_new.index # Drop 0 volume rows df_new = df_new.dropna() - columns = ["date", "open", "high", "low", "close", "volume"] - return list(zip(*[df_new[x].values.tolist() for x in columns])) + return df_new[DEFAULT_DATAFRAME_COLUMNS] + + +def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): + """ + Convert trades from one format to another format. + :param config: Config dictionary + :param convert_from: Source format + :param convert_to: Target format + :param erase: Erase souce data (does not apply if source and target format are identical) + """ + from freqtrade.data.history.idatahandler import get_datahandler + src = get_datahandler(config['datadir'], convert_from) + trg = get_datahandler(config['datadir'], convert_to) + + if 'pairs' not in config: + config['pairs'] = src.trades_get_pairs(config['datadir']) + logger.info(f"Converting trades for {config['pairs']}") + + for pair in config['pairs']: + data = src.trades_load(pair=pair) + logger.info(f"Converting {len(data)} trades for {pair}") + trg.trades_store(pair, data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source Trade data for {pair}.") + src.trades_purge(pair=pair) + + +def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): + """ + Convert OHLCV from one format to another + :param config: Config dictionary + :param convert_from: Source format + :param convert_to: Target format + :param erase: Erase souce data (does not apply if source and target format are identical) + """ + from freqtrade.data.history.idatahandler import get_datahandler + src = get_datahandler(config['datadir'], convert_from) + trg = get_datahandler(config['datadir'], convert_to) + timeframes = config.get('timeframes', [config.get('ticker_interval')]) + logger.info(f"Converting candle (OHLCV) for timeframe {timeframes}") + + if 'pairs' not in config: + config['pairs'] = [] + # Check timeframes or fall back to ticker_interval. + for timeframe in timeframes: + config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'], + timeframe)) + logger.info(f"Converting candle (OHLCV) data for {config['pairs']}") + + for timeframe in timeframes: + for pair in config['pairs']: + data = src.ohlcv_load(pair=pair, timeframe=timeframe, + timerange=None, + fill_missing=False, + drop_incomplete=False, + startup_candles=0) + logger.info(f"Converting {len(data)} candles for {pair}") + trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) + if erase and convert_from != convert_to: + logger.info(f"Deleting source data for {pair} / {timeframe}") + src.ohlcv_purge(pair=pair, timeframe=timeframe) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 7b7159145..7ada4f642 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -1,16 +1,16 @@ """ Dataprovider Responsible to provide data to the bot -including Klines, tickers, historic data +including ticker and orderbook data, live and historical candle (OHLCV) data Common Interface for bot and strategy to access data. """ import logging -from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame from freqtrade.data.history import load_pair_history +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import Exchange from freqtrade.state import RunMode @@ -19,9 +19,10 @@ logger = logging.getLogger(__name__) class DataProvider: - def __init__(self, config: dict, exchange: Exchange) -> None: + def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None: self._config = config self._exchange = exchange + self._pairlists = pairlists def refresh(self, pairlist: List[Tuple[str, str]], @@ -44,10 +45,10 @@ class DataProvider: def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: """ - Get ohlcv data for the given pair as DataFrame + Get candle (OHLCV) data for the given pair as DataFrame Please use the `available_pairs` method to verify which pairs are currently cached. :param pair: pair to get the data for - :param timeframe: Ticker timeframe to get data for + :param timeframe: Timeframe to get data for :param copy: copy dataframe before returning if True. Use False only for read-only operations (where the dataframe is not modified) """ @@ -59,28 +60,28 @@ class DataProvider: def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame: """ - Get stored historic ohlcv data + Get stored historical candle (OHLCV) data :param pair: pair to get the data for :param timeframe: timeframe to get data for """ return load_pair_history(pair=pair, timeframe=timeframe or self._config['ticker_interval'], - datadir=Path(self._config['datadir']) + datadir=self._config['datadir'] ) def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame: """ - Return pair ohlcv data, either live or cached historical -- depending + Return pair candle (OHLCV) data, either live or cached historical -- depending on the runmode. :param pair: pair to get the data for :param timeframe: timeframe to get data for :return: Dataframe for this pair """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - # Get live ohlcv data. + # Get live OHLCV data. data = self.ohlcv(pair=pair, timeframe=timeframe) else: - # Get historic ohlcv data (cached on disk). + # Get historical OHLCV data (cached on disk). data = self.historic_ohlcv(pair=pair, timeframe=timeframe) if len(data) == 0: logger.warning(f"No data found for ({pair}, {timeframe}).") @@ -96,10 +97,14 @@ class DataProvider: def ticker(self, pair: str): """ - Return last ticker data + Return last ticker data from exchange + :param pair: Pair to get the data for + :return: Ticker dict from exchange or empty dict if ticker is not available for the pair """ - # TODO: Implement me - pass + try: + return self._exchange.fetch_ticker(pair) + except DependencyException: + return {} def orderbook(self, pair: str, maximum: int) -> Dict[str, List]: """ @@ -117,3 +122,17 @@ class DataProvider: can be "live", "dry-run", "backtest", "edgecli", "hyperopt" or "other". """ return RunMode(self._config.get('runmode', RunMode.OTHER)) + + def current_whitelist(self) -> List[str]: + """ + fetch latest available whitelist. + + Useful when you have a large whitelist and need to call each pair as an informative pair. + As available pairs does not show whitelist until after informative pairs have been cached. + :return: list of pairs in whitelist + """ + + if self._pairlists: + return self._pairlists.whitelist + else: + raise OperationalException("Dataprovider was not initialized with a pairlist provider.") diff --git a/freqtrade/data/history.py b/freqtrade/data/history.py deleted file mode 100644 index 4c5c0521f..000000000 --- a/freqtrade/data/history.py +++ /dev/null @@ -1,479 +0,0 @@ -""" -Handle historic data (ohlcv). - -Includes: -* load data for a pair (or a list of pairs) from disk -* download data from exchange and store to disk -""" - -import logging -import operator -from copy import deepcopy -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -import arrow -from pandas import DataFrame - -from freqtrade import OperationalException, misc -from freqtrade.configuration import TimeRange -from freqtrade.data.converter import parse_ticker_dataframe, trades_to_ohlcv -from freqtrade.exchange import Exchange, timeframe_to_minutes, timeframe_to_seconds - -logger = logging.getLogger(__name__) - - -def trim_tickerlist(tickerlist: List[Dict], timerange: TimeRange) -> List[Dict]: - """ - Trim tickerlist based on given timerange - """ - if not tickerlist: - return tickerlist - - start_index = 0 - stop_index = len(tickerlist) - - if timerange.starttype == 'date': - while (start_index < len(tickerlist) and - tickerlist[start_index][0] < timerange.startts * 1000): - start_index += 1 - - if timerange.stoptype == 'date': - while (stop_index > 0 and - tickerlist[stop_index-1][0] > timerange.stopts * 1000): - stop_index -= 1 - - if start_index > stop_index: - raise ValueError(f'The timerange [{timerange.startts},{timerange.stopts}] is incorrect') - - return tickerlist[start_index:stop_index] - - -def trim_dataframe(df: DataFrame, timerange: TimeRange, df_date_col: str = 'date') -> DataFrame: - """ - Trim dataframe based on given timerange - :param df: Dataframe to trim - :param timerange: timerange (use start and end date if available) - :param: df_date_col: Column in the dataframe to use as Date column - :return: trimmed dataframe - """ - if timerange.starttype == 'date': - start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) - df = df.loc[df[df_date_col] >= start, :] - if timerange.stoptype == 'date': - stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) - df = df.loc[df[df_date_col] <= stop, :] - return df - - -def load_tickerdata_file(datadir: Path, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None) -> List[Dict]: - """ - Load a pair from file, either .json.gz or .json - :return: tickerlist or None if unsuccessful - """ - filename = pair_data_filename(datadir, pair, timeframe) - pairdata = misc.file_load_json(filename) - if not pairdata: - return [] - - if timerange: - pairdata = trim_tickerlist(pairdata, timerange) - return pairdata - - -def store_tickerdata_file(datadir: Path, pair: str, - timeframe: str, data: list, is_zip: bool = False): - """ - Stores tickerdata to file - """ - filename = pair_data_filename(datadir, pair, timeframe) - misc.file_dump_json(filename, data, is_zip=is_zip) - - -def load_trades_file(datadir: Path, pair: str, - timerange: Optional[TimeRange] = None) -> List[Dict]: - """ - Load a pair from file, either .json.gz or .json - :return: tradelist or empty list if unsuccesful - """ - filename = pair_trades_filename(datadir, pair) - tradesdata = misc.file_load_json(filename) - if not tradesdata: - return [] - - return tradesdata - - -def store_trades_file(datadir: Path, pair: str, - data: list, is_zip: bool = True): - """ - Stores tickerdata to file - """ - filename = pair_trades_filename(datadir, pair) - misc.file_dump_json(filename, data, is_zip=is_zip) - - -def _validate_pairdata(pair, pairdata, timerange: TimeRange): - if timerange.starttype == 'date' and pairdata[0][0] > timerange.startts * 1000: - logger.warning('Missing data at start for pair %s, data starts at %s', - pair, arrow.get(pairdata[0][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - if timerange.stoptype == 'date' and pairdata[-1][0] < timerange.stopts * 1000: - logger.warning('Missing data at end for pair %s, data ends at %s', - pair, arrow.get(pairdata[-1][0] // 1000).strftime('%Y-%m-%d %H:%M:%S')) - - -def load_pair_history(pair: str, - timeframe: str, - datadir: Path, - timerange: Optional[TimeRange] = None, - fill_up_missing: bool = True, - drop_incomplete: bool = True, - startup_candles: int = 0, - ) -> DataFrame: - """ - Load cached ticker history for the given pair. - - :param pair: Pair to load data for - :param timeframe: Ticker timeframe (e.g. "5m") - :param datadir: Path to the data storage location. - :param timerange: Limit data to be loaded to this timerange - :param fill_up_missing: Fill missing values with "No action"-candles - :param drop_incomplete: Drop last candle assuming it may be incomplete. - :param startup_candles: Additional candles to load at the start of the period - :return: DataFrame with ohlcv data, or empty DataFrame - """ - timerange_startup = deepcopy(timerange) - if startup_candles > 0 and timerange_startup: - timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) - - pairdata = load_tickerdata_file(datadir, pair, timeframe, timerange=timerange_startup) - - if pairdata: - if timerange_startup: - _validate_pairdata(pair, pairdata, timerange_startup) - return parse_ticker_dataframe(pairdata, timeframe, pair=pair, - fill_missing=fill_up_missing, - drop_incomplete=drop_incomplete) - else: - logger.warning( - f'No history data for pair: "{pair}", timeframe: {timeframe}. ' - 'Use `freqtrade download-data` to download the data' - ) - return DataFrame() - - -def load_data(datadir: Path, - timeframe: str, - pairs: List[str], - timerange: Optional[TimeRange] = None, - fill_up_missing: bool = True, - startup_candles: int = 0, - fail_without_data: bool = False - ) -> Dict[str, DataFrame]: - """ - Load ticker history data for a list of pairs. - - :param datadir: Path to the data storage location. - :param timeframe: Ticker Timeframe (e.g. "5m") - :param pairs: List of pairs to load - :param timerange: Limit data to be loaded to this timerange - :param fill_up_missing: Fill missing values with "No action"-candles - :param startup_candles: Additional candles to load at the start of the period - :param fail_without_data: Raise OperationalException if no data is found. - :return: dict(:) - """ - result: Dict[str, DataFrame] = {} - if startup_candles > 0 and timerange: - logger.info(f'Using indicator startup period: {startup_candles} ...') - - for pair in pairs: - hist = load_pair_history(pair=pair, timeframe=timeframe, - datadir=datadir, timerange=timerange, - fill_up_missing=fill_up_missing, - startup_candles=startup_candles) - if not hist.empty: - result[pair] = hist - - if fail_without_data and not result: - raise OperationalException("No data found. Terminating.") - return result - - -def refresh_data(datadir: Path, - timeframe: str, - pairs: List[str], - exchange: Exchange, - timerange: Optional[TimeRange] = None, - ) -> None: - """ - Refresh ticker history data for a list of pairs. - - :param datadir: Path to the data storage location. - :param timeframe: Ticker Timeframe (e.g. "5m") - :param pairs: List of pairs to load - :param exchange: Exchange object - :param timerange: Limit data to be loaded to this timerange - """ - for pair in pairs: - _download_pair_history(pair=pair, timeframe=timeframe, - datadir=datadir, timerange=timerange, - exchange=exchange) - - -def pair_data_filename(datadir: Path, pair: str, timeframe: str) -> Path: - pair_s = pair.replace("/", "_") - filename = datadir.joinpath(f'{pair_s}-{timeframe}.json') - return filename - - -def pair_trades_filename(datadir: Path, pair: str) -> Path: - pair_s = pair.replace("/", "_") - filename = datadir.joinpath(f'{pair_s}-trades.json.gz') - return filename - - -def _load_cached_data_for_updating(datadir: Path, pair: str, timeframe: str, - timerange: Optional[TimeRange]) -> Tuple[List[Any], - Optional[int]]: - """ - Load cached data to download more data. - If timerange is passed in, checks whether data from an before the stored data will be - downloaded. - If that's the case then what's available should be completely overwritten. - Only used by download_pair_history(). - """ - - since_ms = None - - # user sets timerange, so find the start time - if timerange: - if timerange.starttype == 'date': - since_ms = timerange.startts * 1000 - elif timerange.stoptype == 'line': - num_minutes = timerange.stopts * timeframe_to_minutes(timeframe) - since_ms = arrow.utcnow().shift(minutes=num_minutes).timestamp * 1000 - - # read the cached file - # Intentionally don't pass timerange in - since we need to load the full dataset. - data = load_tickerdata_file(datadir, pair, timeframe) - # remove the last item, could be incomplete candle - if data: - data.pop() - else: - data = [] - - if data: - if since_ms and since_ms < data[0][0]: - # Earlier data than existing data requested, redownload all - data = [] - else: - # a part of the data was already downloaded, so download unexist data only - since_ms = data[-1][0] + 1 - - return (data, since_ms) - - -def _download_pair_history(datadir: Path, - exchange: Exchange, - pair: str, - timeframe: str = '5m', - timerange: Optional[TimeRange] = None) -> bool: - """ - Download latest candles from the exchange for the pair and timeframe passed in parameters - The data is downloaded starting from the last correct data that - exists in a cache. If timerange starts earlier than the data in the cache, - the full data will be redownloaded - - Based on @Rybolov work: https://github.com/rybolov/freqtrade-data - - :param pair: pair to download - :param timeframe: Ticker Timeframe (e.g 5m) - :param timerange: range of time to download - :return: bool with success state - """ - try: - logger.info( - f'Download history data for pair: "{pair}", timeframe: {timeframe} ' - f'and store in {datadir}.' - ) - - data, since_ms = _load_cached_data_for_updating(datadir, pair, timeframe, timerange) - - logger.debug("Current Start: %s", misc.format_ms_time(data[1][0]) if data else 'None') - logger.debug("Current End: %s", misc.format_ms_time(data[-1][0]) if data else 'None') - - # Default since_ms to 30 days if nothing is given - new_data = exchange.get_historic_ohlcv(pair=pair, - timeframe=timeframe, - since_ms=since_ms if since_ms else - int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000 - ) - data.extend(new_data) - - logger.debug("New Start: %s", misc.format_ms_time(data[0][0])) - logger.debug("New End: %s", misc.format_ms_time(data[-1][0])) - - store_tickerdata_file(datadir, pair, timeframe, data=data) - return True - - except Exception as e: - logger.error( - f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}. ' - f'Error: {e}' - ) - return False - - -def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], - datadir: Path, timerange: Optional[TimeRange] = None, - erase=False) -> List[str]: - """ - Refresh stored ohlcv data for backtesting and hyperopt operations. - Used by freqtrade download-data subcommand. - :return: List of pairs that are not available. - """ - pairs_not_available = [] - for pair in pairs: - if pair not in exchange.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - for timeframe in timeframes: - - dl_file = pair_data_filename(datadir, pair, timeframe) - if erase and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}, interval {timeframe}.') - dl_file.unlink() - - logger.info(f'Downloading pair {pair}, interval {timeframe}.') - _download_pair_history(datadir=datadir, exchange=exchange, - pair=pair, timeframe=str(timeframe), - timerange=timerange) - return pairs_not_available - - -def _download_trades_history(datadir: Path, - exchange: Exchange, - pair: str, - timerange: Optional[TimeRange] = None) -> bool: - """ - Download trade history from the exchange. - Appends to previously downloaded trades data. - """ - try: - - since = timerange.startts * 1000 if timerange and timerange.starttype == 'date' else None - - trades = load_trades_file(datadir, pair) - - from_id = trades[-1]['id'] if trades else None - - logger.debug("Current Start: %s", trades[0]['datetime'] if trades else 'None') - logger.debug("Current End: %s", trades[-1]['datetime'] if trades else 'None') - - # Default since_ms to 30 days if nothing is given - new_trades = exchange.get_historic_trades(pair=pair, - since=since if since else - int(arrow.utcnow().shift( - days=-30).float_timestamp) * 1000, - from_id=from_id, - ) - trades.extend(new_trades[1]) - store_trades_file(datadir, pair, trades) - - logger.debug("New Start: %s", trades[0]['datetime']) - logger.debug("New End: %s", trades[-1]['datetime']) - logger.info(f"New Amount of trades: {len(trades)}") - return True - - except Exception as e: - logger.error( - f'Failed to download historic trades for pair: "{pair}". ' - f'Error: {e}' - ) - return False - - -def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, erase=False) -> List[str]: - """ - Refresh stored trades data for backtesting and hyperopt operations. - Used by freqtrade download-data subcommand. - :return: List of pairs that are not available. - """ - pairs_not_available = [] - for pair in pairs: - if pair not in exchange.markets: - pairs_not_available.append(pair) - logger.info(f"Skipping pair {pair}...") - continue - - dl_file = pair_trades_filename(datadir, pair) - if erase and dl_file.exists(): - logger.info( - f'Deleting existing data for pair {pair}.') - dl_file.unlink() - - logger.info(f'Downloading trades for pair {pair}.') - _download_trades_history(datadir=datadir, exchange=exchange, - pair=pair, - timerange=timerange) - return pairs_not_available - - -def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], - datadir: Path, timerange: TimeRange, erase=False) -> None: - """ - Convert stored trades data to ohlcv data - """ - for pair in pairs: - trades = load_trades_file(datadir, pair) - for timeframe in timeframes: - ohlcv_file = pair_data_filename(datadir, pair, timeframe) - if erase and ohlcv_file.exists(): - logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') - ohlcv_file.unlink() - ohlcv = trades_to_ohlcv(trades, timeframe) - # Store ohlcv - store_tickerdata_file(datadir, pair, timeframe, data=ohlcv) - - -def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: - """ - Get the maximum common timerange for the given backtest data. - - :param data: dictionary with preprocessed backtesting data - :return: tuple containing min_date, max_date - """ - timeranges = [ - (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) - for frame in data.values() - ] - return (min(timeranges, key=operator.itemgetter(0))[0], - max(timeranges, key=operator.itemgetter(1))[1]) - - -def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime, - max_date: datetime, timeframe_min: int) -> bool: - """ - Validates preprocessed backtesting data for missing values and shows warnings about it that. - - :param data: preprocessed backtesting data (as DataFrame) - :param pair: pair used for log output. - :param min_date: start-date of the data - :param max_date: end-date of the data - :param timeframe_min: ticker Timeframe in minutes - """ - # total difference in minutes / timeframe-minutes - expected_frames = int((max_date - min_date).total_seconds() // 60 // timeframe_min) - found_missing = False - dflen = len(data) - if dflen < expected_frames: - found_missing = True - logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values", - pair, expected_frames, dflen, expected_frames - dflen) - return found_missing diff --git a/freqtrade/data/history/__init__.py b/freqtrade/data/history/__init__.py new file mode 100644 index 000000000..23f635a98 --- /dev/null +++ b/freqtrade/data/history/__init__.py @@ -0,0 +1,14 @@ +""" +Handle historic data (ohlcv). + +Includes: +* load data for a pair (or a list of pairs) from disk +* download data from exchange and store to disk +""" + +from .history_utils import (convert_trades_to_ohlcv, # noqa: F401 + get_timerange, load_data, load_pair_history, + refresh_backtest_ohlcv_data, + refresh_backtest_trades_data, refresh_data, + validate_backtest_data) +from .idatahandler import get_datahandler # noqa: F401 diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py new file mode 100644 index 000000000..4f3f75a87 --- /dev/null +++ b/freqtrade/data/history/history_utils.py @@ -0,0 +1,391 @@ +import logging +import operator +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import arrow +from pandas import DataFrame + +from freqtrade.configuration import TimeRange +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.data.converter import (ohlcv_to_dataframe, + trades_remove_duplicates, + trades_to_ohlcv) +from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import Exchange +from freqtrade.misc import format_ms_time + +logger = logging.getLogger(__name__) + + +def load_pair_history(pair: str, + timeframe: str, + datadir: Path, *, + timerange: Optional[TimeRange] = None, + fill_up_missing: bool = True, + drop_incomplete: bool = True, + startup_candles: int = 0, + data_format: str = None, + data_handler: IDataHandler = None, + ) -> DataFrame: + """ + Load cached ohlcv history for the given pair. + + :param pair: Pair to load data for + :param timeframe: Timeframe (e.g. "5m") + :param datadir: Path to the data storage location. + :param data_format: Format of the data. Ignored if data_handler is set. + :param timerange: Limit data to be loaded to this timerange + :param fill_up_missing: Fill missing values with "No action"-candles + :param drop_incomplete: Drop last candle assuming it may be incomplete. + :param startup_candles: Additional candles to load at the start of the period + :param data_handler: Initialized data-handler to use. + Will be initialized from data_format if not set + :return: DataFrame with ohlcv data, or empty DataFrame + """ + data_handler = get_datahandler(datadir, data_format, data_handler) + + return data_handler.ohlcv_load(pair=pair, + timeframe=timeframe, + timerange=timerange, + fill_missing=fill_up_missing, + drop_incomplete=drop_incomplete, + startup_candles=startup_candles, + ) + + +def load_data(datadir: Path, + timeframe: str, + pairs: List[str], *, + timerange: Optional[TimeRange] = None, + fill_up_missing: bool = True, + startup_candles: int = 0, + fail_without_data: bool = False, + data_format: str = 'json', + ) -> Dict[str, DataFrame]: + """ + Load ohlcv history data for a list of pairs. + + :param datadir: Path to the data storage location. + :param timeframe: Timeframe (e.g. "5m") + :param pairs: List of pairs to load + :param timerange: Limit data to be loaded to this timerange + :param fill_up_missing: Fill missing values with "No action"-candles + :param startup_candles: Additional candles to load at the start of the period + :param fail_without_data: Raise OperationalException if no data is found. + :param data_format: Data format which should be used. Defaults to json + :return: dict(:) + """ + result: Dict[str, DataFrame] = {} + if startup_candles > 0 and timerange: + logger.info(f'Using indicator startup period: {startup_candles} ...') + + data_handler = get_datahandler(datadir, data_format) + + for pair in pairs: + hist = load_pair_history(pair=pair, timeframe=timeframe, + datadir=datadir, timerange=timerange, + fill_up_missing=fill_up_missing, + startup_candles=startup_candles, + data_handler=data_handler + ) + if not hist.empty: + result[pair] = hist + + if fail_without_data and not result: + raise OperationalException("No data found. Terminating.") + return result + + +def refresh_data(datadir: Path, + timeframe: str, + pairs: List[str], + exchange: Exchange, + data_format: str = None, + timerange: Optional[TimeRange] = None, + ) -> None: + """ + Refresh ohlcv history data for a list of pairs. + + :param datadir: Path to the data storage location. + :param timeframe: Timeframe (e.g. "5m") + :param pairs: List of pairs to load + :param exchange: Exchange object + :param timerange: Limit data to be loaded to this timerange + """ + data_handler = get_datahandler(datadir, data_format) + for pair in pairs: + _download_pair_history(pair=pair, timeframe=timeframe, + datadir=datadir, timerange=timerange, + exchange=exchange, data_handler=data_handler) + + +def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], + data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]: + """ + Load cached data to download more data. + If timerange is passed in, checks whether data from an before the stored data will be + downloaded. + If that's the case then what's available should be completely overwritten. + Otherwise downloads always start at the end of the available data to avoid data gaps. + Note: Only used by download_pair_history(). + """ + start = None + if timerange: + if timerange.starttype == 'date': + # TODO: convert to date for conversion + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + + # Intentionally don't pass timerange in - since we need to load the full dataset. + data = data_handler.ohlcv_load(pair, timeframe=timeframe, + timerange=None, fill_missing=False, + drop_incomplete=True, warn_no_data=False) + if not data.empty: + if start and start < data.iloc[0]['date']: + # Earlier data than existing data requested, redownload all + data = DataFrame(columns=DEFAULT_DATAFRAME_COLUMNS) + else: + start = data.iloc[-1]['date'] + + start_ms = int(start.timestamp() * 1000) if start else None + return data, start_ms + + +def _download_pair_history(datadir: Path, + exchange: Exchange, + pair: str, *, + timeframe: str = '5m', + timerange: Optional[TimeRange] = None, + data_handler: IDataHandler = None) -> bool: + """ + Download latest candles from the exchange for the pair and timeframe passed in parameters + The data is downloaded starting from the last correct data that + exists in a cache. If timerange starts earlier than the data in the cache, + the full data will be redownloaded + + Based on @Rybolov work: https://github.com/rybolov/freqtrade-data + + :param pair: pair to download + :param timeframe: Timeframe (e.g "5m") + :param timerange: range of time to download + :return: bool with success state + """ + data_handler = get_datahandler(datadir, data_handler=data_handler) + + try: + logger.info( + f'Download history data for pair: "{pair}", timeframe: {timeframe} ' + f'and store in {datadir}.' + ) + + # data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange) + data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, + data_handler=data_handler) + + logger.debug("Current Start: %s", + f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + logger.debug("Current End: %s", + f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + + # Default since_ms to 30 days if nothing is given + new_data = exchange.get_historic_ohlcv(pair=pair, + timeframe=timeframe, + since_ms=since_ms if since_ms else + int(arrow.utcnow().shift( + days=-30).float_timestamp) * 1000 + ) + # TODO: Maybe move parsing to exchange class (?) + new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, + fill_missing=False, drop_incomplete=True) + if data.empty: + data = new_dataframe + else: + data = data.append(new_dataframe) + + logger.debug("New Start: %s", + f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + logger.debug("New End: %s", + f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') + + data_handler.ohlcv_store(pair, timeframe, data=data) + return True + + except Exception as e: + logger.error( + f'Failed to download history data for pair: "{pair}", timeframe: {timeframe}. ' + f'Error: {e}' + ) + return False + + +def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], + datadir: Path, timerange: Optional[TimeRange] = None, + erase: bool = False, data_format: str = None) -> List[str]: + """ + Refresh stored ohlcv data for backtesting and hyperopt operations. + Used by freqtrade download-data subcommand. + :return: List of pairs that are not available. + """ + pairs_not_available = [] + data_handler = get_datahandler(datadir, data_format) + for pair in pairs: + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + for timeframe in timeframes: + + if erase: + if data_handler.ohlcv_purge(pair, timeframe): + logger.info( + f'Deleting existing data for pair {pair}, interval {timeframe}.') + + logger.info(f'Downloading pair {pair}, interval {timeframe}.') + _download_pair_history(datadir=datadir, exchange=exchange, + pair=pair, timeframe=str(timeframe), + timerange=timerange, data_handler=data_handler) + return pairs_not_available + + +def _download_trades_history(exchange: Exchange, + pair: str, *, + timerange: Optional[TimeRange] = None, + data_handler: IDataHandler + ) -> bool: + """ + Download trade history from the exchange. + Appends to previously downloaded trades data. + """ + try: + + since = timerange.startts * 1000 if \ + (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift( + days=-30).float_timestamp) * 1000 + + trades = data_handler.trades_load(pair) + + # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id + + from_id = trades[-1][1] if trades else None + if trades and since < trades[-1][0]: + # Reset since to the last available point + # - 5 seconds (to ensure we're getting all trades) + since = trades[-1][0] - (5 * 1000) + logger.info(f"Using last trade date -5s - Downloading trades for {pair} " + f"since: {format_ms_time(since)}.") + + logger.debug(f"Current Start: {format_ms_time(trades[0][0]) if trades else 'None'}") + logger.debug(f"Current End: {format_ms_time(trades[-1][0]) if trades else 'None'}") + logger.info(f"Current Amount of trades: {len(trades)}") + + # Default since_ms to 30 days if nothing is given + new_trades = exchange.get_historic_trades(pair=pair, + since=since, + from_id=from_id, + ) + trades.extend(new_trades[1]) + # Remove duplicates to make sure we're not storing data we don't need + trades = trades_remove_duplicates(trades) + data_handler.trades_store(pair, data=trades) + + logger.debug(f"New Start: {format_ms_time(trades[0][0])}") + logger.debug(f"New End: {format_ms_time(trades[-1][0])}") + logger.info(f"New Amount of trades: {len(trades)}") + return True + + except Exception as e: + logger.error( + f'Failed to download historic trades for pair: "{pair}". ' + f'Error: {e}' + ) + return False + + +def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, + timerange: TimeRange, erase: bool = False, + data_format: str = 'jsongz') -> List[str]: + """ + Refresh stored trades data for backtesting and hyperopt operations. + Used by freqtrade download-data subcommand. + :return: List of pairs that are not available. + """ + pairs_not_available = [] + data_handler = get_datahandler(datadir, data_format=data_format) + for pair in pairs: + if pair not in exchange.markets: + pairs_not_available.append(pair) + logger.info(f"Skipping pair {pair}...") + continue + + if erase: + if data_handler.trades_purge(pair): + logger.info(f'Deleting existing data for pair {pair}.') + + logger.info(f'Downloading trades for pair {pair}.') + _download_trades_history(exchange=exchange, + pair=pair, + timerange=timerange, + data_handler=data_handler) + return pairs_not_available + + +def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], + datadir: Path, timerange: TimeRange, erase: bool = False, + data_format_ohlcv: str = 'json', + data_format_trades: str = 'jsongz') -> None: + """ + Convert stored trades data to ohlcv data + """ + data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) + data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) + + for pair in pairs: + trades = data_handler_trades.trades_load(pair) + for timeframe in timeframes: + if erase: + if data_handler_ohlcv.ohlcv_purge(pair, timeframe): + logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') + ohlcv = trades_to_ohlcv(trades, timeframe) + # Store ohlcv + data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) + + +def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]: + """ + Get the maximum common timerange for the given backtest data. + + :param data: dictionary with preprocessed backtesting data + :return: tuple containing min_date, max_date + """ + timeranges = [ + (arrow.get(frame['date'].min()), arrow.get(frame['date'].max())) + for frame in data.values() + ] + return (min(timeranges, key=operator.itemgetter(0))[0], + max(timeranges, key=operator.itemgetter(1))[1]) + + +def validate_backtest_data(data: DataFrame, pair: str, min_date: datetime, + max_date: datetime, timeframe_min: int) -> bool: + """ + Validates preprocessed backtesting data for missing values and shows warnings about it that. + + :param data: preprocessed backtesting data (as DataFrame) + :param pair: pair used for log output. + :param min_date: start-date of the data + :param max_date: end-date of the data + :param timeframe_min: Timeframe in minutes + """ + # total difference in minutes / timeframe-minutes + expected_frames = int((max_date - min_date).total_seconds() // 60 // timeframe_min) + found_missing = False + dflen = len(data) + if dflen < expected_frames: + found_missing = True + logger.warning("%s has missing frames: expected %s, got %s, that's %s missing values", + pair, expected_frames, dflen, expected_frames - dflen) + return found_missing diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py new file mode 100644 index 000000000..d5d7c16db --- /dev/null +++ b/freqtrade/data/history/idatahandler.py @@ -0,0 +1,248 @@ +""" +Abstract datahandler interface. +It's subclasses handle and storing data from disk. + +""" +import logging +from abc import ABC, abstractclassmethod, abstractmethod +from copy import deepcopy +from datetime import datetime, timezone +from pathlib import Path +from typing import List, Optional, Type + +from pandas import DataFrame + +from freqtrade.configuration import TimeRange +from freqtrade.data.converter import (clean_ohlcv_dataframe, + trades_remove_duplicates, trim_dataframe) +from freqtrade.exchange import timeframe_to_seconds + +logger = logging.getLogger(__name__) + +# Type for trades list +TradeList = List[List] + + +class IDataHandler(ABC): + + def __init__(self, datadir: Path) -> None: + self._datadir = datadir + + @abstractclassmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs with ohlcv data available in this datadir + for the specified timeframe + :param datadir: Directory to search for ohlcv files + :param timeframe: Timeframe to search pairs for + :return: List of Pairs + """ + + @abstractmethod + def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Store data in json format "values". + format looks as follows: + [[,,,,]] + :param pair: Pair - used to generate filename + :timeframe: Timeframe - used to generate filename + :data: Dataframe containing OHLCV data + :return: None + """ + + @abstractmethod + def _ohlcv_load(self, pair: str, timeframe: str, + timerange: Optional[TimeRange] = None, + ) -> DataFrame: + """ + Internal method used to load data for one pair from disk. + Implements the loading and conversion to a Pandas dataframe. + Timerange trimming and dataframe validation happens outside of this method. + :param pair: Pair to load data + :param timeframe: Timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. + :return: DataFrame with ohlcv data, or empty DataFrame + """ + + @abstractmethod + def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :param timeframe: Timeframe (e.g. "5m") + :return: True when deleted, false if file did not exist. + """ + + @abstractmethod + def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Append data to existing data structures + :param pair: Pair + :param timeframe: Timeframe this ohlcv data is for + :param data: Data to append. + """ + + @abstractclassmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs for which trade data is available in this + :param datadir: Directory to search for ohlcv files + :return: List of Pairs + """ + + @abstractmethod + def trades_store(self, pair: str, data: TradeList) -> None: + """ + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + + @abstractmethod + def trades_append(self, pair: str, data: TradeList): + """ + Append data to existing files + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + + @abstractmethod + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from file, either .json.gz or .json + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + + @abstractmethod + def trades_purge(self, pair: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :return: True when deleted, false if file did not exist. + """ + + def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from file, either .json.gz or .json + Removes duplicates in the process. + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + + def ohlcv_load(self, pair, timeframe: str, + timerange: Optional[TimeRange] = None, + fill_missing: bool = True, + drop_incomplete: bool = True, + startup_candles: int = 0, + warn_no_data: bool = True + ) -> DataFrame: + """ + Load cached candle (OHLCV) data for the given pair. + + :param pair: Pair to load data for + :param timeframe: Timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange + :param fill_missing: Fill missing values with "No action"-candles + :param drop_incomplete: Drop last candle assuming it may be incomplete. + :param startup_candles: Additional candles to load at the start of the period + :param warn_no_data: Log a warning message when no data is found + :return: DataFrame with ohlcv data, or empty DataFrame + """ + # Fix startup period + timerange_startup = deepcopy(timerange) + if startup_candles > 0 and timerange_startup: + timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) + + pairdf = self._ohlcv_load(pair, timeframe, + timerange=timerange_startup) + if self._check_empty_df(pairdf, pair, timeframe, warn_no_data): + return pairdf + else: + enddate = pairdf.iloc[-1]['date'] + + if timerange_startup: + self._validate_pairdata(pair, pairdf, timerange_startup) + pairdf = trim_dataframe(pairdf, timerange_startup) + if self._check_empty_df(pairdf, pair, timeframe, warn_no_data): + return pairdf + + # incomplete candles should only be dropped if we didn't trim the end beforehand. + pairdf = clean_ohlcv_dataframe(pairdf, timeframe, + pair=pair, + fill_missing=fill_missing, + drop_incomplete=(drop_incomplete and + enddate == pairdf.iloc[-1]['date'])) + self._check_empty_df(pairdf, pair, timeframe, warn_no_data) + return pairdf + + def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str, warn_no_data: bool): + """ + Warn on empty dataframe + """ + if pairdf.empty: + if warn_no_data: + logger.warning( + f'No history data for pair: "{pair}", timeframe: {timeframe}. ' + 'Use `freqtrade download-data` to download the data' + ) + return True + return False + + def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange): + """ + Validates pairdata for missing data at start end end and logs warnings. + :param pairdata: Dataframe to validate + :param timerange: Timerange specified for start and end dates + """ + + if timerange.starttype == 'date': + start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc) + if pairdata.iloc[0]['date'] > start: + logger.warning(f"Missing data at start for pair {pair}, " + f"data starts at {pairdata.iloc[0]['date']:%Y-%m-%d %H:%M:%S}") + if timerange.stoptype == 'date': + stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc) + if pairdata.iloc[-1]['date'] < stop: + logger.warning(f"Missing data at end for pair {pair}, " + f"data ends at {pairdata.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}") + + +def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: + """ + Get datahandler class. + Could be done using Resolvers, but since this may be called often and resolvers + are rather expensive, doing this directly should improve performance. + :param datatype: datatype to use. + :return: Datahandler class + """ + + if datatype == 'json': + from .jsondatahandler import JsonDataHandler + return JsonDataHandler + elif datatype == 'jsongz': + from .jsondatahandler import JsonGzDataHandler + return JsonGzDataHandler + else: + raise ValueError(f"No datahandler for datatype {datatype} available.") + + +def get_datahandler(datadir: Path, data_format: str = None, + data_handler: IDataHandler = None) -> IDataHandler: + """ + :param datadir: Folder to save data + :data_format: dataformat to use + :data_handler: returns this datahandler if it exists or initializes a new one + """ + + if not data_handler: + HandlerClass = get_datahandlerclass(data_format or 'json') + data_handler = HandlerClass(datadir) + return data_handler diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py new file mode 100644 index 000000000..01320f129 --- /dev/null +++ b/freqtrade/data/history/jsondatahandler.py @@ -0,0 +1,191 @@ +import logging +import re +from pathlib import Path +from typing import List, Optional + +import numpy as np +from pandas import DataFrame, read_json, to_datetime + +from freqtrade import misc +from freqtrade.configuration import TimeRange +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS +from freqtrade.data.converter import trades_dict_to_list + +from .idatahandler import IDataHandler, TradeList + +logger = logging.getLogger(__name__) + + +class JsonDataHandler(IDataHandler): + + _use_zip = False + _columns = DEFAULT_DATAFRAME_COLUMNS + + @classmethod + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + """ + Returns a list of all pairs with ohlcv data available in this datadir + for the specified timeframe + :param datadir: Directory to search for ohlcv files + :param timeframe: Timeframe to search pairs for + :return: List of Pairs + """ + + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name) + for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] + # Check if regex found something and only return these results + return [match[0].replace('_', '/') for match in _tmp if match] + + def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Store data in json format "values". + format looks as follows: + [[,,,,]] + :param pair: Pair - used to generate filename + :timeframe: Timeframe - used to generate filename + :data: Dataframe containing OHLCV data + :return: None + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + _data = data.copy() + # Convert date to int + _data['date'] = _data['date'].astype(np.int64) // 1000 // 1000 + + # Reset index, select only appropriate columns and save as json + _data.reset_index(drop=True).loc[:, self._columns].to_json( + filename, orient="values", + compression='gzip' if self._use_zip else None) + + def _ohlcv_load(self, pair: str, timeframe: str, + timerange: Optional[TimeRange] = None, + ) -> DataFrame: + """ + Internal method used to load data for one pair from disk. + Implements the loading and conversion to a Pandas dataframe. + Timerange trimming and dataframe validation happens outside of this method. + :param pair: Pair to load data + :param timeframe: Timeframe (e.g. "5m") + :param timerange: Limit data to be loaded to this timerange. + Optionally implemented by subclasses to avoid loading + all data where possible. + :return: DataFrame with ohlcv data, or empty DataFrame + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if not filename.exists(): + return DataFrame(columns=self._columns) + pairdata = read_json(filename, orient='values') + pairdata.columns = self._columns + pairdata = pairdata.astype(dtype={'open': 'float', 'high': 'float', + 'low': 'float', 'close': 'float', 'volume': 'float'}) + pairdata['date'] = to_datetime(pairdata['date'], + unit='ms', + utc=True, + infer_datetime_format=True) + return pairdata + + def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :param timeframe: Timeframe (e.g. "5m") + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_data_filename(self._datadir, pair, timeframe) + if filename.exists(): + filename.unlink() + return True + return False + + def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: + """ + Append data to existing data structures + :param pair: Pair + :param timeframe: Timeframe this ohlcv data is for + :param data: Data to append. + """ + raise NotImplementedError() + + @classmethod + def trades_get_pairs(cls, datadir: Path) -> List[str]: + """ + Returns a list of all pairs for which trade data is available in this + :param datadir: Directory to search for ohlcv files + :return: List of Pairs + """ + _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) + for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] + # Check if regex found something and only return these results to avoid exceptions. + return [match[0].replace('_', '/') for match in _tmp if match] + + def trades_store(self, pair: str, data: TradeList) -> None: + """ + Store trades data (list of Dicts) to file + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + filename = self._pair_trades_filename(self._datadir, pair) + misc.file_dump_json(filename, data, is_zip=self._use_zip) + + def trades_append(self, pair: str, data: TradeList): + """ + Append data to existing files + :param pair: Pair - used for filename + :param data: List of Lists containing trade data, + column sequence as in DEFAULT_TRADES_COLUMNS + """ + raise NotImplementedError() + + def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + """ + Load a pair from file, either .json.gz or .json + # TODO: respect timerange ... + :param pair: Load trades for this pair + :param timerange: Timerange to load trades for - currently not implemented + :return: List of trades + """ + filename = self._pair_trades_filename(self._datadir, pair) + tradesdata = misc.file_load_json(filename) + + if not tradesdata: + return [] + + if isinstance(tradesdata[0], dict): + # Convert trades dict to list + logger.info("Old trades format detected - converting") + tradesdata = trades_dict_to_list(tradesdata) + pass + return tradesdata + + def trades_purge(self, pair: str) -> bool: + """ + Remove data for this pair + :param pair: Delete data for this pair. + :return: True when deleted, false if file did not exist. + """ + filename = self._pair_trades_filename(self._datadir, pair) + if filename.exists(): + filename.unlink() + return True + return False + + @classmethod + def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') + return filename + + @classmethod + def _get_file_extension(cls): + return "json.gz" if cls._use_zip else "json" + + @classmethod + def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + pair_s = misc.pair_to_filename(pair) + filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') + return filename + + +class JsonGzDataHandler(JsonDataHandler): + + _use_zip = True diff --git a/freqtrade/edge/__init__.py b/freqtrade/edge/__init__.py index e56071a98..d275a80e3 100644 --- a/freqtrade/edge/__init__.py +++ b/freqtrade/edge/__init__.py @@ -1,463 +1 @@ -# pragma pylint: disable=W0603 -""" Edge positioning package """ -import logging -from pathlib import Path -from typing import Any, Dict, NamedTuple - -import arrow -import numpy as np -import utils_find_1st as utf1st -from pandas import DataFrame - -from freqtrade import constants, OperationalException -from freqtrade.configuration import TimeRange -from freqtrade.data import history -from freqtrade.strategy.interface import SellType - - -logger = logging.getLogger(__name__) - - -class PairInfo(NamedTuple): - stoploss: float - winrate: float - risk_reward_ratio: float - required_risk_reward: float - expectancy: float - nb_trades: int - avg_trade_duration: float - - -class Edge: - """ - Calculates Win Rate, Risk Reward Ratio, Expectancy - against historical data for a give set of markets and a strategy - it then adjusts stoploss and position size accordingly - and force it into the strategy - Author: https://github.com/mishaker - """ - - config: Dict = {} - _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - - def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: - - self.config = config - self.exchange = exchange - self.strategy = strategy - - self.edge_config = self.config.get('edge', {}) - self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs - self._final_pairs: list = [] - - # checking max_open_trades. it should be -1 as with Edge - # the number of trades is determined by position size - if self.config['max_open_trades'] != float('inf'): - logger.critical('max_open_trades should be -1 in config !') - - if self.config['stake_amount'] != constants.UNLIMITED_STAKE_AMOUNT: - raise OperationalException('Edge works only with unlimited stake amount') - - self._capital_percentage: float = self.edge_config.get('capital_available_percentage') - self._allowed_risk: float = self.edge_config.get('allowed_risk') - self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) - self._last_updated: int = 0 # Timestamp of pairs last updated time - self._refresh_pairs = True - - self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) - self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) - self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) - - # calculating stoploss range - self._stoploss_range = np.arange( - self._stoploss_range_min, - self._stoploss_range_max, - self._stoploss_range_step - ) - - self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift( - days=-1 * self._since_number_of_days).format('YYYYMMDD')) - if config.get('fee'): - self.fee = config['fee'] - else: - self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) - - def calculate(self) -> bool: - pairs = self.config['exchange']['pair_whitelist'] - heartbeat = self.edge_config.get('process_throttle_secs') - - if (self._last_updated > 0) and ( - self._last_updated + heartbeat > arrow.utcnow().timestamp): - return False - - data: Dict[str, Any] = {} - logger.info('Using stake_currency: %s ...', self.config['stake_currency']) - logger.info('Using local backtesting data (using whitelist in given config) ...') - - if self._refresh_pairs: - history.refresh_data( - datadir=Path(self.config['datadir']), - pairs=pairs, - exchange=self.exchange, - timeframe=self.strategy.ticker_interval, - timerange=self._timerange, - ) - - data = history.load_data( - datadir=Path(self.config['datadir']), - pairs=pairs, - timeframe=self.strategy.ticker_interval, - timerange=self._timerange, - startup_candles=self.strategy.startup_candle_count, - ) - - if not data: - # Reinitializing cached pairs - self._cached_pairs = {} - logger.critical("No data found. Edge is stopped ...") - return False - - preprocessed = self.strategy.tickerdata_to_dataframe(data) - - # Print timeframe - min_date, max_date = history.get_timerange(preprocessed) - logger.info( - 'Measuring data from %s up to %s (%s days) ...', - min_date.isoformat(), - max_date.isoformat(), - (max_date - min_date).days - ) - headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] - - trades: list = [] - for pair, pair_data in preprocessed.items(): - # Sorting dataframe by date and reset index - pair_data = pair_data.sort_values(by=['date']) - pair_data = pair_data.reset_index(drop=True) - - ticker_data = self.strategy.advise_sell( - self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - - trades += self._find_trades_for_stoploss_range(ticker_data, pair, self._stoploss_range) - - # If no trade found then exit - if len(trades) == 0: - logger.info("No trades found.") - return False - - # Fill missing, calculable columns, profit, duration , abs etc. - trades_df = self._fill_calculable_fields(DataFrame(trades)) - self._cached_pairs = self._process_expectancy(trades_df) - self._last_updated = arrow.utcnow().timestamp - - return True - - def stake_amount(self, pair: str, free_capital: float, - total_capital: float, capital_in_trade: float) -> float: - stoploss = self.stoploss(pair) - available_capital = (total_capital + capital_in_trade) * self._capital_percentage - allowed_capital_at_risk = available_capital * self._allowed_risk - max_position_size = abs(allowed_capital_at_risk / stoploss) - position_size = min(max_position_size, free_capital) - if pair in self._cached_pairs: - logger.info( - 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' - ' capital in trade: %s, free capital: %s, total capital: %s,' - ' stoploss: %s, available capital: %s.', - self._cached_pairs[pair].winrate, - self._cached_pairs[pair].expectancy, - position_size, pair, - capital_in_trade, free_capital, total_capital, - stoploss, available_capital - ) - return round(position_size, 15) - - def stoploss(self, pair: str) -> float: - if pair in self._cached_pairs: - return self._cached_pairs[pair].stoploss - else: - logger.warning('tried to access stoploss of a non-existing pair, ' - 'strategy stoploss is returned instead.') - return self.strategy.stoploss - - def adjust(self, pairs) -> list: - """ - Filters out and sorts "pairs" according to Edge calculated pairs - """ - final = [] - for pair, info in self._cached_pairs.items(): - if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ - info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ - pair in pairs: - final.append(pair) - - if self._final_pairs != final: - self._final_pairs = final - if self._final_pairs: - logger.info( - 'Minimum expectancy and minimum winrate are met only for %s,' - ' so other pairs are filtered out.', - self._final_pairs - ) - else: - logger.info( - 'Edge removed all pairs as no pair with minimum expectancy ' - 'and minimum winrate was found !' - ) - - return self._final_pairs - - def accepted_pairs(self) -> list: - """ - return a list of accepted pairs along with their winrate, expectancy and stoploss - """ - final = [] - for pair, info in self._cached_pairs.items(): - if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ - info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)): - final.append({ - 'Pair': pair, - 'Winrate': info.winrate, - 'Expectancy': info.expectancy, - 'Stoploss': info.stoploss, - }) - return final - - def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: - """ - The result frame contains a number of columns that are calculable - from other columns. These are left blank till all rows are added, - to be populated in single vector calls. - - Columns to be populated are: - - Profit - - trade duration - - profit abs - :param result Dataframe - :return: result Dataframe - """ - - # stake and fees - # stake = 0.015 - # 0.05% is 0.0005 - # fee = 0.001 - - # we set stake amount to an arbitrary amount. - # as it doesn't change the calculation. - # all returned values are relative. they are percentages. - stake = 0.015 - fee = self.fee - open_fee = fee / 2 - close_fee = fee / 2 - - result['trade_duration'] = result['close_time'] - result['open_time'] - - result['trade_duration'] = result['trade_duration'].map( - lambda x: int(x.total_seconds() / 60)) - - # Spends, Takes, Profit, Absolute Profit - - # Buy Price - result['buy_vol'] = stake / result['open_rate'] # How many target are we buying - result['buy_fee'] = stake * open_fee - result['buy_spend'] = stake + result['buy_fee'] # How much we're spending - - # Sell price - result['sell_sum'] = result['buy_vol'] * result['close_rate'] - result['sell_fee'] = result['sell_sum'] * close_fee - result['sell_take'] = result['sell_sum'] - result['sell_fee'] - - # profit_percent - result['profit_percent'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend'] - - # Absolute profit - result['profit_abs'] = result['sell_take'] - result['buy_spend'] - - return result - - def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: - """ - This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs - The calulation will be done per pair and per strategy. - """ - # Removing pairs having less than min_trades_number - min_trades_number = self.edge_config.get('min_trade_number', 10) - results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) - ################################### - - # Removing outliers (Only Pumps) from the dataset - # The method to detect outliers is to calculate standard deviation - # Then every value more than (standard deviation + 2*average) is out (pump) - # - # Removing Pumps - if self.edge_config.get('remove_pumps', False): - results = results.groupby(['pair', 'stoploss']).apply( - lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) - ########################################################################## - - # Removing trades having a duration more than X minutes (set in config) - max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440) - results = results[results.trade_duration < max_trade_duration] - ####################################################################### - - if results.empty: - return {} - - groupby_aggregator = { - 'profit_abs': [ - ('nb_trades', 'count'), # number of all trades - ('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades - ('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades - ('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades - ], - 'trade_duration': [('avg_trade_duration', 'mean')] - } - - # Group by (pair and stoploss) by applying above aggregator - df = results.groupby(['pair', 'stoploss'])['profit_abs', 'trade_duration'].agg( - groupby_aggregator).reset_index(col_level=1) - - # Dropping level 0 as we don't need it - df.columns = df.columns.droplevel(0) - - # Calculating number of losing trades, average win and average loss - df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] - df['average_win'] = df['profit_sum'] / df['nb_win_trades'] - df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] - - # Win rate = number of profitable trades / number of trades - df['winrate'] = df['nb_win_trades'] / df['nb_trades'] - - # risk_reward_ratio = average win / average loss - df['risk_reward_ratio'] = df['average_win'] / df['average_loss'] - - # required_risk_reward = (1 / winrate) - 1 - df['required_risk_reward'] = (1 / df['winrate']) - 1 - - # expectancy = (risk_reward_ratio * winrate) - (lossrate) - df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate']) - - # sort by expectancy and stoploss - df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( - 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() - - final = {} - for x in df.itertuples(): - final[x.pair] = PairInfo( - x.stoploss, - x.winrate, - x.risk_reward_ratio, - x.required_risk_reward, - x.expectancy, - x.nb_trades, - x.avg_trade_duration - ) - - # Returning a list of pairs in order of "expectancy" - return final - - def _find_trades_for_stoploss_range(self, ticker_data, pair, stoploss_range): - buy_column = ticker_data['buy'].values - sell_column = ticker_data['sell'].values - date_column = ticker_data['date'].values - ohlc_columns = ticker_data[['open', 'high', 'low', 'close']].values - - result: list = [] - for stoploss in stoploss_range: - result += self._detect_next_stop_or_sell_point( - buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair - ) - - return result - - def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, - ohlc_columns, stoploss, pair): - """ - Iterate through ohlc_columns in order to find the next trade - Next trade opens from the first buy signal noticed to - The sell or stoploss signal after it. - It then cuts OHLC, buy_column, sell_column and date_column. - Cut from (the exit trade index) + 1. - - Author: https://github.com/mishaker - """ - - result: list = [] - start_point = 0 - - while True: - open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) - - # Return empty if we don't find trade entry (i.e. buy==1) or - # we find a buy but at the end of array - if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: - break - else: - # When a buy signal is seen, - # trade opens in reality on the next candle - open_trade_index += 1 - - stop_price_percentage = stoploss + 1 - open_price = ohlc_columns[open_trade_index, 0] - stop_price = (open_price * stop_price_percentage) - - # Searching for the index where stoploss is hit - stop_index = utf1st.find_1st( - ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller) - - # If we don't find it then we assume stop_index will be far in future (infinite number) - if stop_index == -1: - stop_index = float('inf') - - # Searching for the index where sell is hit - sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal) - - # If we don't find it then we assume sell_index will be far in future (infinite number) - if sell_index == -1: - sell_index = float('inf') - - # Check if we don't find any stop or sell point (in that case trade remains open) - # It is not interesting for Edge to consider it so we simply ignore the trade - # And stop iterating there is no more entry - if stop_index == sell_index == float('inf'): - break - - if stop_index <= sell_index: - exit_index = open_trade_index + stop_index - exit_type = SellType.STOP_LOSS - exit_price = stop_price - elif stop_index > sell_index: - # If exit is SELL then we exit at the next candle - exit_index = open_trade_index + sell_index + 1 - - # Check if we have the next candle - if len(ohlc_columns) - 1 < exit_index: - break - - exit_type = SellType.SELL_SIGNAL - exit_price = ohlc_columns[exit_index, 0] - - trade = {'pair': pair, - 'stoploss': stoploss, - 'profit_percent': '', - 'profit_abs': '', - 'open_time': date_column[open_trade_index], - 'close_time': date_column[exit_index], - 'open_index': start_point + open_trade_index, - 'close_index': start_point + exit_index, - 'trade_duration': '', - 'open_rate': round(open_price, 15), - 'close_rate': round(exit_price, 15), - 'exit_type': exit_type - } - - result.append(trade) - - # Giving a view of exit_index till the end of array - buy_column = buy_column[exit_index:] - sell_column = sell_column[exit_index:] - date_column = date_column[exit_index:] - ohlc_columns = ohlc_columns[exit_index:] - start_point += exit_index - - return result +from .edge_positioning import Edge, PairInfo # noqa: F401 diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py new file mode 100644 index 000000000..c19d4552a --- /dev/null +++ b/freqtrade/edge/edge_positioning.py @@ -0,0 +1,454 @@ +# pragma pylint: disable=W0603 +""" Edge positioning package """ +import logging +from typing import Any, Dict, List, NamedTuple + +import arrow +import numpy as np +import utils_find_1st as utf1st +from pandas import DataFrame + +from freqtrade.configuration import TimeRange +from freqtrade.constants import UNLIMITED_STAKE_AMOUNT +from freqtrade.exceptions import OperationalException +from freqtrade.data.history import get_timerange, load_data, refresh_data +from freqtrade.strategy.interface import SellType + +logger = logging.getLogger(__name__) + + +class PairInfo(NamedTuple): + stoploss: float + winrate: float + risk_reward_ratio: float + required_risk_reward: float + expectancy: float + nb_trades: int + avg_trade_duration: float + + +class Edge: + """ + Calculates Win Rate, Risk Reward Ratio, Expectancy + against historical data for a give set of markets and a strategy + it then adjusts stoploss and position size accordingly + and force it into the strategy + Author: https://github.com/mishaker + """ + + config: Dict = {} + _cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + + def __init__(self, config: Dict[str, Any], exchange, strategy) -> None: + + self.config = config + self.exchange = exchange + self.strategy = strategy + + self.edge_config = self.config.get('edge', {}) + self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs + self._final_pairs: list = [] + + # checking max_open_trades. it should be -1 as with Edge + # the number of trades is determined by position size + if self.config['max_open_trades'] != float('inf'): + logger.critical('max_open_trades should be -1 in config !') + + if self.config['stake_amount'] != UNLIMITED_STAKE_AMOUNT: + raise OperationalException('Edge works only with unlimited stake amount') + + # Deprecated capital_available_percentage. Will use tradable_balance_ratio in the future. + self._capital_percentage: float = self.edge_config.get( + 'capital_available_percentage', self.config['tradable_balance_ratio']) + self._allowed_risk: float = self.edge_config.get('allowed_risk') + self._since_number_of_days: int = self.edge_config.get('calculate_since_number_of_days', 14) + self._last_updated: int = 0 # Timestamp of pairs last updated time + self._refresh_pairs = True + + self._stoploss_range_min = float(self.edge_config.get('stoploss_range_min', -0.01)) + self._stoploss_range_max = float(self.edge_config.get('stoploss_range_max', -0.05)) + self._stoploss_range_step = float(self.edge_config.get('stoploss_range_step', -0.001)) + + # calculating stoploss range + self._stoploss_range = np.arange( + self._stoploss_range_min, + self._stoploss_range_max, + self._stoploss_range_step + ) + + self._timerange: TimeRange = TimeRange.parse_timerange("%s-" % arrow.now().shift( + days=-1 * self._since_number_of_days).format('YYYYMMDD')) + if config.get('fee'): + self.fee = config['fee'] + else: + self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) + + def calculate(self) -> bool: + pairs = self.config['exchange']['pair_whitelist'] + heartbeat = self.edge_config.get('process_throttle_secs') + + if (self._last_updated > 0) and ( + self._last_updated + heartbeat > arrow.utcnow().timestamp): + return False + + data: Dict[str, Any] = {} + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) + logger.info('Using local backtesting data (using whitelist in given config) ...') + + if self._refresh_pairs: + refresh_data( + datadir=self.config['datadir'], + pairs=pairs, + exchange=self.exchange, + timeframe=self.strategy.ticker_interval, + timerange=self._timerange, + ) + + data = load_data( + datadir=self.config['datadir'], + pairs=pairs, + timeframe=self.strategy.ticker_interval, + timerange=self._timerange, + startup_candles=self.strategy.startup_candle_count, + data_format=self.config.get('dataformat_ohlcv', 'json'), + ) + + if not data: + # Reinitializing cached pairs + self._cached_pairs = {} + logger.critical("No data found. Edge is stopped ...") + return False + + preprocessed = self.strategy.ohlcvdata_to_dataframe(data) + + # Print timeframe + min_date, max_date = get_timerange(preprocessed) + logger.info( + 'Measuring data from %s up to %s (%s days) ...', + min_date.isoformat(), + max_date.isoformat(), + (max_date - min_date).days + ) + headers = ['date', 'buy', 'open', 'close', 'sell', 'high', 'low'] + + trades: list = [] + for pair, pair_data in preprocessed.items(): + # Sorting dataframe by date and reset index + pair_data = pair_data.sort_values(by=['date']) + pair_data = pair_data.reset_index(drop=True) + + df_analyzed = self.strategy.advise_sell( + self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() + + trades += self._find_trades_for_stoploss_range(df_analyzed, pair, self._stoploss_range) + + # If no trade found then exit + if len(trades) == 0: + logger.info("No trades found.") + return False + + # Fill missing, calculable columns, profit, duration , abs etc. + trades_df = self._fill_calculable_fields(DataFrame(trades)) + self._cached_pairs = self._process_expectancy(trades_df) + self._last_updated = arrow.utcnow().timestamp + + return True + + def stake_amount(self, pair: str, free_capital: float, + total_capital: float, capital_in_trade: float) -> float: + stoploss = self.stoploss(pair) + available_capital = (total_capital + capital_in_trade) * self._capital_percentage + allowed_capital_at_risk = available_capital * self._allowed_risk + max_position_size = abs(allowed_capital_at_risk / stoploss) + position_size = min(max_position_size, free_capital) + if pair in self._cached_pairs: + logger.info( + 'winrate: %s, expectancy: %s, position size: %s, pair: %s,' + ' capital in trade: %s, free capital: %s, total capital: %s,' + ' stoploss: %s, available capital: %s.', + self._cached_pairs[pair].winrate, + self._cached_pairs[pair].expectancy, + position_size, pair, + capital_in_trade, free_capital, total_capital, + stoploss, available_capital + ) + return round(position_size, 15) + + def stoploss(self, pair: str) -> float: + if pair in self._cached_pairs: + return self._cached_pairs[pair].stoploss + else: + logger.warning('tried to access stoploss of a non-existing pair, ' + 'strategy stoploss is returned instead.') + return self.strategy.stoploss + + def adjust(self, pairs: List[str]) -> list: + """ + Filters out and sorts "pairs" according to Edge calculated pairs + """ + final = [] + for pair, info in self._cached_pairs.items(): + if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)) and \ + pair in pairs: + final.append(pair) + + if self._final_pairs != final: + self._final_pairs = final + if self._final_pairs: + logger.info( + 'Minimum expectancy and minimum winrate are met only for %s,' + ' so other pairs are filtered out.', + self._final_pairs + ) + else: + logger.info( + 'Edge removed all pairs as no pair with minimum expectancy ' + 'and minimum winrate was found !' + ) + + return self._final_pairs + + def accepted_pairs(self) -> list: + """ + return a list of accepted pairs along with their winrate, expectancy and stoploss + """ + final = [] + for pair, info in self._cached_pairs.items(): + if info.expectancy > float(self.edge_config.get('minimum_expectancy', 0.2)) and \ + info.winrate > float(self.edge_config.get('minimum_winrate', 0.60)): + final.append({ + 'Pair': pair, + 'Winrate': info.winrate, + 'Expectancy': info.expectancy, + 'Stoploss': info.stoploss, + }) + return final + + def _fill_calculable_fields(self, result: DataFrame) -> DataFrame: + """ + The result frame contains a number of columns that are calculable + from other columns. These are left blank till all rows are added, + to be populated in single vector calls. + + Columns to be populated are: + - Profit + - trade duration + - profit abs + :param result Dataframe + :return: result Dataframe + """ + # We set stake amount to an arbitrary amount, as it doesn't change the calculation. + # All returned values are relative, they are defined as ratios. + stake = 0.015 + + result['trade_duration'] = result['close_time'] - result['open_time'] + + result['trade_duration'] = result['trade_duration'].map( + lambda x: int(x.total_seconds() / 60)) + + # Spends, Takes, Profit, Absolute Profit + + # Buy Price + result['buy_vol'] = stake / result['open_rate'] # How many target are we buying + result['buy_fee'] = stake * self.fee + result['buy_spend'] = stake + result['buy_fee'] # How much we're spending + + # Sell price + result['sell_sum'] = result['buy_vol'] * result['close_rate'] + result['sell_fee'] = result['sell_sum'] * self.fee + result['sell_take'] = result['sell_sum'] - result['sell_fee'] + + # profit_ratio + result['profit_ratio'] = (result['sell_take'] - result['buy_spend']) / result['buy_spend'] + + # Absolute profit + result['profit_abs'] = result['sell_take'] - result['buy_spend'] + + return result + + def _process_expectancy(self, results: DataFrame) -> Dict[str, Any]: + """ + This calculates WinRate, Required Risk Reward, Risk Reward and Expectancy of all pairs + The calulation will be done per pair and per strategy. + """ + # Removing pairs having less than min_trades_number + min_trades_number = self.edge_config.get('min_trade_number', 10) + results = results.groupby(['pair', 'stoploss']).filter(lambda x: len(x) > min_trades_number) + ################################### + + # Removing outliers (Only Pumps) from the dataset + # The method to detect outliers is to calculate standard deviation + # Then every value more than (standard deviation + 2*average) is out (pump) + # + # Removing Pumps + if self.edge_config.get('remove_pumps', False): + results = results.groupby(['pair', 'stoploss']).apply( + lambda x: x[x['profit_abs'] < 2 * x['profit_abs'].std() + x['profit_abs'].mean()]) + ########################################################################## + + # Removing trades having a duration more than X minutes (set in config) + max_trade_duration = self.edge_config.get('max_trade_duration_minute', 1440) + results = results[results.trade_duration < max_trade_duration] + ####################################################################### + + if results.empty: + return {} + + groupby_aggregator = { + 'profit_abs': [ + ('nb_trades', 'count'), # number of all trades + ('profit_sum', lambda x: x[x > 0].sum()), # cumulative profit of all winning trades + ('loss_sum', lambda x: abs(x[x < 0].sum())), # cumulative loss of all losing trades + ('nb_win_trades', lambda x: x[x > 0].count()) # number of winning trades + ], + 'trade_duration': [('avg_trade_duration', 'mean')] + } + + # Group by (pair and stoploss) by applying above aggregator + df = results.groupby(['pair', 'stoploss'])[['profit_abs', 'trade_duration']].agg( + groupby_aggregator).reset_index(col_level=1) + + # Dropping level 0 as we don't need it + df.columns = df.columns.droplevel(0) + + # Calculating number of losing trades, average win and average loss + df['nb_loss_trades'] = df['nb_trades'] - df['nb_win_trades'] + df['average_win'] = df['profit_sum'] / df['nb_win_trades'] + df['average_loss'] = df['loss_sum'] / df['nb_loss_trades'] + + # Win rate = number of profitable trades / number of trades + df['winrate'] = df['nb_win_trades'] / df['nb_trades'] + + # risk_reward_ratio = average win / average loss + df['risk_reward_ratio'] = df['average_win'] / df['average_loss'] + + # required_risk_reward = (1 / winrate) - 1 + df['required_risk_reward'] = (1 / df['winrate']) - 1 + + # expectancy = (risk_reward_ratio * winrate) - (lossrate) + df['expectancy'] = (df['risk_reward_ratio'] * df['winrate']) - (1 - df['winrate']) + + # sort by expectancy and stoploss + df = df.sort_values(by=['expectancy', 'stoploss'], ascending=False).groupby( + 'pair').first().sort_values(by=['expectancy'], ascending=False).reset_index() + + final = {} + for x in df.itertuples(): + final[x.pair] = PairInfo( + x.stoploss, + x.winrate, + x.risk_reward_ratio, + x.required_risk_reward, + x.expectancy, + x.nb_trades, + x.avg_trade_duration + ) + + # Returning a list of pairs in order of "expectancy" + return final + + def _find_trades_for_stoploss_range(self, df, pair, stoploss_range): + buy_column = df['buy'].values + sell_column = df['sell'].values + date_column = df['date'].values + ohlc_columns = df[['open', 'high', 'low', 'close']].values + + result: list = [] + for stoploss in stoploss_range: + result += self._detect_next_stop_or_sell_point( + buy_column, sell_column, date_column, ohlc_columns, round(stoploss, 6), pair + ) + + return result + + def _detect_next_stop_or_sell_point(self, buy_column, sell_column, date_column, + ohlc_columns, stoploss, pair): + """ + Iterate through ohlc_columns in order to find the next trade + Next trade opens from the first buy signal noticed to + The sell or stoploss signal after it. + It then cuts OHLC, buy_column, sell_column and date_column. + Cut from (the exit trade index) + 1. + + Author: https://github.com/mishaker + """ + + result: list = [] + start_point = 0 + + while True: + open_trade_index = utf1st.find_1st(buy_column, 1, utf1st.cmp_equal) + + # Return empty if we don't find trade entry (i.e. buy==1) or + # we find a buy but at the end of array + if open_trade_index == -1 or open_trade_index == len(buy_column) - 1: + break + else: + # When a buy signal is seen, + # trade opens in reality on the next candle + open_trade_index += 1 + + open_price = ohlc_columns[open_trade_index, 0] + stop_price = (open_price * (stoploss + 1)) + + # Searching for the index where stoploss is hit + stop_index = utf1st.find_1st( + ohlc_columns[open_trade_index:, 2], stop_price, utf1st.cmp_smaller) + + # If we don't find it then we assume stop_index will be far in future (infinite number) + if stop_index == -1: + stop_index = float('inf') + + # Searching for the index where sell is hit + sell_index = utf1st.find_1st(sell_column[open_trade_index:], 1, utf1st.cmp_equal) + + # If we don't find it then we assume sell_index will be far in future (infinite number) + if sell_index == -1: + sell_index = float('inf') + + # Check if we don't find any stop or sell point (in that case trade remains open) + # It is not interesting for Edge to consider it so we simply ignore the trade + # And stop iterating there is no more entry + if stop_index == sell_index == float('inf'): + break + + if stop_index <= sell_index: + exit_index = open_trade_index + stop_index + exit_type = SellType.STOP_LOSS + exit_price = stop_price + elif stop_index > sell_index: + # If exit is SELL then we exit at the next candle + exit_index = open_trade_index + sell_index + 1 + + # Check if we have the next candle + if len(ohlc_columns) - 1 < exit_index: + break + + exit_type = SellType.SELL_SIGNAL + exit_price = ohlc_columns[exit_index, 0] + + trade = {'pair': pair, + 'stoploss': stoploss, + 'profit_ratio': '', + 'profit_abs': '', + 'open_time': date_column[open_trade_index], + 'close_time': date_column[exit_index], + 'open_index': start_point + open_trade_index, + 'close_index': start_point + exit_index, + 'trade_duration': '', + 'open_rate': round(open_price, 15), + 'close_rate': round(exit_price, 15), + 'exit_type': exit_type + } + + result.append(trade) + + # Giving a view of exit_index till the end of array + buy_column = buy_column[exit_index:] + sell_column = sell_column[exit_index:] + date_column = date_column[exit_index:] + ohlc_columns = ohlc_columns[exit_index:] + start_point += exit_index + + return result diff --git a/freqtrade/exceptions.py b/freqtrade/exceptions.py new file mode 100644 index 000000000..553a691ef --- /dev/null +++ b/freqtrade/exceptions.py @@ -0,0 +1,44 @@ + + +class FreqtradeException(Exception): + """ + Freqtrade base exception. Handled at the outermost level. + All other exception types are subclasses of this exception type. + """ + + +class OperationalException(FreqtradeException): + """ + Requires manual intervention and will stop the bot. + Most of the time, this is caused by an invalid Configuration. + """ + + +class DependencyException(FreqtradeException): + """ + Indicates that an assumed dependency is not met. + This could happen when there is currently not enough money on the account. + """ + + +class InvalidOrderException(FreqtradeException): + """ + This is returned when the order is not valid. Example: + If stoploss on exchange order is hit, then trying to cancel the order + should return this exception. + """ + + +class TemporaryError(FreqtradeException): + """ + Temporary network or exchange related error. + This could happen when an exchange is congested, unavailable, or the user + has networking problems. Usually resolves itself after a time. + """ + + +class StrategyError(FreqtradeException): + """ + Errors with custom user-code deteced. + Usually caused by errors in the strategy. + """ diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index df18bca02..a39f8f5df 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,18 +1,20 @@ -from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS # noqa: F401 -from freqtrade.exchange.exchange import Exchange # noqa: F401 -from freqtrade.exchange.exchange import (get_exchange_bad_reason, # noqa: F401 +# flake8: noqa: F401 +from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS +from freqtrade.exchange.exchange import Exchange +from freqtrade.exchange.exchange import (get_exchange_bad_reason, is_exchange_bad, is_exchange_known_ccxt, is_exchange_officially_supported, ccxt_exchanges, available_exchanges) -from freqtrade.exchange.exchange import (timeframe_to_seconds, # noqa: F401 +from freqtrade.exchange.exchange import (timeframe_to_seconds, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date) -from freqtrade.exchange.exchange import (market_is_active, # noqa: F401 +from freqtrade.exchange.exchange import (market_is_active, symbol_is_pair) -from freqtrade.exchange.kraken import Kraken # noqa: F401 -from freqtrade.exchange.binance import Binance # noqa: F401 -from freqtrade.exchange.bibox import Bibox # noqa: F401 +from freqtrade.exchange.kraken import Kraken +from freqtrade.exchange.binance import Binance +from freqtrade.exchange.bibox import Bibox +from freqtrade.exchange.ftx import Ftx diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index b5507981f..37183dc2c 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -4,8 +4,8 @@ from typing import Dict import ccxt -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange import Exchange logger = logging.getLogger(__name__) @@ -32,16 +32,26 @@ class Binance(Exchange): return super().get_order_book(pair, limit) - def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + return order['type'] == 'stop_loss_limit' and stop_loss > float(order['info']['stopPrice']) + + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: """ creates a stoploss limit order. this stoploss-limit is binance-specific. It may work with a limited number of other exchanges, but this has not been tested yet. - """ + # Limit price threshold: As limit price should always be below stop-price + limit_price_pct = order_types.get('stoploss_on_exchange_limit_ratio', 0.99) + rate = stop_price * limit_price_pct + ordertype = "stop_loss_limit" - stop_price = self.symbol_price_prec(pair, stop_price) + stop_price = self.price_to_precision(pair, stop_price) # Ensure rate is less than stop price if stop_price <= rate: @@ -57,12 +67,12 @@ class Binance(Exchange): params = self._params.copy() params.update({'stopPrice': stop_price}) - amount = self.symbol_amount_prec(pair, amount) + amount = self.amount_to_precision(pair, amount) - rate = self.symbol_price_prec(pair, rate) + rate = self.price_to_precision(pair, rate) - order = self._api.create_order(pair, ordertype, 'sell', - amount, rate, params) + order = self._api.create_order(symbol=pair, type=ordertype, side='sell', + amount=amount, price=rate, params=params) logger.info('stoploss limit order added for %s. ' 'stop price: %s. limit: %s', pair, stop_price, rate) return order diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index ed30b95c7..b38ed35a3 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -1,6 +1,6 @@ import logging -from freqtrade import DependencyException, TemporaryError +from freqtrade.exceptions import DependencyException, TemporaryError logger = logging.getLogger(__name__) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e4e7aacce..68022662a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -7,21 +7,25 @@ import inspect import logging from copy import deepcopy from datetime import datetime, timezone -from math import ceil, floor +from math import ceil from random import randint from typing import Any, Dict, List, Optional, Tuple import arrow import ccxt import ccxt.async_support as ccxt_async -from ccxt.base.decimal_to_precision import ROUND_DOWN, ROUND_UP +from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, + TRUNCATE, decimal_to_precision) from pandas import DataFrame -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) -from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async -from freqtrade.misc import deep_merge_dicts +from freqtrade.misc import deep_merge_dicts, safe_value_fallback + +CcxtModuleType = Any + logger = logging.getLogger(__name__) @@ -50,7 +54,7 @@ class Exchange: } _ft_has: Dict = {} - def __init__(self, config: dict, validate: bool = True) -> None: + def __init__(self, config: Dict[str, Any], validate: bool = True) -> None: """ Initializes this module with the given config, it does basic validation whether the specified exchange and pairs are valid. @@ -61,8 +65,6 @@ class Exchange: self._config.update(config) - self._cached_ticker: Dict[str, Any] = {} - # Holds last candle refreshed time of each pair self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} # Timestamp of last markets refresh @@ -116,6 +118,7 @@ class Exchange: self._load_markets() # Check if all pairs are available + self.validate_stakecurrency(config['stake_currency']) self.validate_pairs(config['exchange']['pair_whitelist']) self.validate_ordertypes(config.get('order_types', {})) self.validate_order_time_in_force(config.get('order_time_in_force', {})) @@ -133,7 +136,7 @@ class Exchange: if self._api_async and inspect.iscoroutinefunction(self._api_async.close): asyncio.get_event_loop().run_until_complete(self._api_async.close()) - def _init_ccxt(self, exchange_config: dict, ccxt_module=ccxt, + def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, ccxt_kwargs: dict = None) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid @@ -188,6 +191,11 @@ class Exchange: self._load_markets() return self._api.markets + @property + def precisionMode(self) -> str: + """exchange ccxt precisionMode""" + return self._api.precisionMode + def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None, pairs_only: bool = False, active_only: bool = False) -> Dict: """ @@ -210,13 +218,32 @@ class Exchange: markets = {k: v for k, v in markets.items() if market_is_active(v)} return markets - def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame: + def get_quote_currencies(self) -> List[str]: + """ + Return a list of supported quote currencies + """ + markets = self.markets + return sorted(set([x['quote'] for _, x in markets.items()])) + + def get_pair_quote_currency(self, pair: str) -> str: + """ + Return a pair's quote currency + """ + return self.markets.get(pair, {}).get('quote', '') + + def get_pair_base_currency(self, pair: str) -> str: + """ + Return a pair's quote currency + """ + return self.markets.get(pair, {}).get('base', '') + + def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: if pair_interval in self._klines: return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] else: return DataFrame() - def set_sandbox(self, api, exchange_config: dict, name: str): + def set_sandbox(self, api: ccxt.Exchange, exchange_config: dict, name: str) -> None: if exchange_config.get('sandbox'): if api.urls.get('test'): api.urls['api'] = api.urls['test'] @@ -226,7 +253,7 @@ class Exchange: "Please check your config.json") raise OperationalException(f'Exchange {name} does not provide a sandbox api') - def _load_async_markets(self, reload=False) -> None: + def _load_async_markets(self, reload: bool = False) -> None: try: if self._api_async: asyncio.get_event_loop().run_until_complete( @@ -259,18 +286,30 @@ class Exchange: except ccxt.BaseError: logger.exception("Could not reload markets.") + def validate_stakecurrency(self, stake_currency: str) -> None: + """ + Checks stake-currency against available currencies on the exchange. + :param stake_currency: Stake-currency to validate + :raise: OperationalException if stake-currency is not available. + """ + quote_currencies = self.get_quote_currencies() + if stake_currency not in quote_currencies: + raise OperationalException( + f"{stake_currency} is not available as stake on {self.name}. " + f"Available currencies are: {', '.join(quote_currencies)}") + def validate_pairs(self, pairs: List[str]) -> None: """ Checks if all given pairs are tradable on the current exchange. - Raises OperationalException if one pair is not available. :param pairs: list of pairs + :raise: OperationalException if one pair is not available :return: None """ if not self.markets: logger.warning('Unable to validate pairs (assuming they are correct).') return - + invalid_pairs = [] for pair in pairs: # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs # TODO: add a support for having coins in BTC/USDT format @@ -278,14 +317,29 @@ class Exchange: raise OperationalException( f'Pair {pair} is not available on {self.name}. ' f'Please remove {pair} from your whitelist.') - elif self.markets[pair].get('info', {}).get('IsRestricted', False): + + # From ccxt Documentation: + # markets.info: An associative array of non-common market properties, + # including fees, rates, limits and other general market information. + # The internal info array is different for each particular market, + # its contents depend on the exchange. + # It can also be a string or similar ... so we need to verify that first. + elif (isinstance(self.markets[pair].get('info', None), dict) + and self.markets[pair].get('info', {}).get('IsRestricted', False)): # Warn users about restricted pairs in whitelist. # We cannot determine reliably if Users are affected. logger.warning(f"Pair {pair} is restricted for some users on this exchange." f"Please check if you are impacted by this restriction " f"on the exchange and eventually remove {pair} from your whitelist.") + if (self._config['stake_currency'] and + self.get_pair_quote_currency(pair) != self._config['stake_currency']): + invalid_pairs.append(pair) + if invalid_pairs: + raise OperationalException( + f"Stake-currency '{self._config['stake_currency']}' not compatible with " + f"pair-whitelist. Please remove the following pairs: {invalid_pairs}") - def get_valid_pair_combination(self, curr_1, curr_2) -> str: + def get_valid_pair_combination(self, curr_1: str, curr_2: str) -> str: """ Get valid pair combination of curr_1 and curr_2 by trying both combinations. """ @@ -296,7 +350,7 @@ class Exchange: def validate_timeframes(self, timeframe: Optional[str]) -> None: """ - Checks if ticker interval from config is a supported timeframe on the exchange + Check if timeframe from config is a supported timeframe on the exchange """ if not hasattr(self._api, "timeframes") or self._api.timeframes is None: # If timeframes attribute is missing (or is None), the exchange probably @@ -309,7 +363,10 @@ class Exchange: if timeframe and (timeframe not in self.timeframes): raise OperationalException( - f"Invalid ticker interval '{timeframe}'. This exchange supports: {self.timeframes}") + f"Invalid timeframe '{timeframe}'. This exchange supports: {self.timeframes}") + + if timeframe and timeframe_to_minutes(timeframe) < 1: + raise OperationalException("Timeframes < 1m are currently not supported by Freqtrade.") def validate_ordertypes(self, order_types: Dict) -> None: """ @@ -335,7 +392,7 @@ class Exchange: raise OperationalException( f'Time in force policies are not supported for {self.name} yet.') - def validate_required_startup_candles(self, startup_candles) -> None: + def validate_required_startup_candles(self, startup_candles: int) -> None: """ Checks if required startup_candles is more than ohlcv_candle_limit. Requires a grace-period of 5 candles - so a startup-period up to 494 is allowed by default. @@ -354,58 +411,91 @@ class Exchange: """ return endpoint in self._api.has and self._api.has[endpoint] - def symbol_amount_prec(self, pair, amount: float): + def amount_to_precision(self, pair: str, amount: float) -> float: ''' Returns the amount to buy or sell to a precision the Exchange accepts - Rounded down + Reimplementation of ccxt internal methods - ensuring we can test the result is correct + based on our definitions. ''' if self.markets[pair]['precision']['amount']: - symbol_prec = self.markets[pair]['precision']['amount'] - big_amount = amount * pow(10, symbol_prec) - amount = floor(big_amount) / pow(10, symbol_prec) + amount = float(decimal_to_precision(amount, rounding_mode=TRUNCATE, + precision=self.markets[pair]['precision']['amount'], + counting_mode=self.precisionMode, + )) + return amount - def symbol_price_prec(self, pair, price: float): + def price_to_precision(self, pair: str, price: float) -> float: ''' - Returns the price buying or selling with to the precision the Exchange accepts + Returns the price rounded up to the precision the Exchange accepts. + Partial Reimplementation of ccxt internal method decimal_to_precision(), + which does not support rounding up + TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and + align with amount_to_precision(). Rounds up ''' if self.markets[pair]['precision']['price']: - symbol_prec = self.markets[pair]['precision']['price'] - big_price = price * pow(10, symbol_prec) - price = ceil(big_price) / pow(10, symbol_prec) + # price = float(decimal_to_precision(price, rounding_mode=ROUND, + # precision=self.markets[pair]['precision']['price'], + # counting_mode=self.precisionMode, + # )) + if self.precisionMode == TICK_SIZE: + precision = self.markets[pair]['precision']['price'] + missing = price % precision + if missing != 0: + price = price - missing + precision + else: + symbol_prec = self.markets[pair]['precision']['price'] + big_price = price * pow(10, symbol_prec) + price = ceil(big_price) / pow(10, symbol_prec) return price + def price_get_one_pip(self, pair: str, price: float) -> float: + """ + Get's the "1 pip" value for this pair. + Used in PriceFilter to calculate the 1pip movements. + """ + precision = self.markets[pair]['precision']['price'] + if self.precisionMode == TICK_SIZE: + return precision + else: + return 1 / pow(10, precision) + def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict[str, Any]: order_id = f'dry_run_{side}_{randint(0, 10**6)}' - _amount = self.symbol_amount_prec(pair, amount) + _amount = self.amount_to_precision(pair, amount) dry_order = { "id": order_id, 'pair': pair, 'price': rate, 'amount': _amount, - "cost": _amount * rate, + 'cost': _amount * rate, 'type': ordertype, 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), 'status': "closed" if ordertype == "market" else "open", 'fee': None, - "info": {} + 'info': {} } - self._store_dry_order(dry_order) + self._store_dry_order(dry_order, pair) # Copy order and close it - so the returned order is open unless it's a market order return dry_order - def _store_dry_order(self, dry_order: Dict) -> None: + def _store_dry_order(self, dry_order: Dict, pair: str) -> None: closed_order = dry_order.copy() - if closed_order["type"] in ["market", "limit"]: + if closed_order['type'] in ["market", "limit"]: closed_order.update({ - "status": "closed", - "filled": closed_order["amount"], - "remaining": 0 - }) + 'status': 'closed', + 'filled': closed_order['amount'], + 'remaining': 0, + 'fee': { + 'currency': self.get_pair_quote_currency(pair), + 'cost': dry_order['cost'] * self.get_fee(pair), + 'rate': self.get_fee(pair) + } + }) if closed_order["type"] in ["stop_loss_limit"]: closed_order["info"].update({"stopPrice": closed_order["price"]}) self._dry_run_open_orders[closed_order["id"]] = closed_order @@ -414,13 +504,13 @@ class Exchange: rate: float, params: Dict = {}) -> Dict: try: # Set the precision for amount and price(rate) as accepted by the exchange - amount = self.symbol_amount_prec(pair, amount) + amount = self.amount_to_precision(pair, amount) needs_price = (ordertype != 'market' or self._api.options.get("createMarketBuyOrderRequiresPrice", False)) - rate = self.symbol_price_prec(pair, rate) if needs_price else None + rate_for_order = self.price_to_precision(pair, rate) if needs_price else None return self._api.create_order(pair, ordertype, side, - amount, rate, params) + amount, rate_for_order, params) except ccxt.InsufficientFunds as e: raise DependencyException( @@ -439,7 +529,7 @@ class Exchange: raise OperationalException(e) from e def buy(self, pair: str, ordertype: str, amount: float, - rate: float, time_in_force) -> Dict: + rate: float, time_in_force: str) -> Dict: if self._config['dry_run']: dry_order = self.dry_run_order(pair, ordertype, "buy", amount, rate) @@ -452,7 +542,7 @@ class Exchange: return self.create_order(pair, ordertype, 'buy', amount, rate, params) def sell(self, pair: str, ordertype: str, amount: float, - rate: float, time_in_force='gtc') -> Dict: + rate: float, time_in_force: str = 'gtc') -> Dict: if self._config['dry_run']: dry_order = self.dry_run_order(pair, ordertype, "sell", amount, rate) @@ -464,9 +554,17 @@ class Exchange: return self.create_order(pair, ordertype, 'sell', amount, rate, params) - def stoploss_limit(self, pair: str, amount: float, stop_price: float, rate: float) -> Dict: + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: """ - creates a stoploss limit order. + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + raise OperationalException(f"stoploss is not implemented for {self.name}.") + + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: + """ + creates a stoploss order. + The precise ordertype is determined by the order_types dict or exchange default. Since ccxt does not unify stoploss-limit orders yet, this needs to be implemented in each exchange's subclass. The exception below should never raise, since we disallow @@ -474,7 +572,7 @@ class Exchange: Note: Changes to this interface need to be applied to all sub-classes too. """ - raise OperationalException(f"stoploss_limit is not implemented for {self.name}.") + raise OperationalException(f"stoploss is not implemented for {self.name}.") @retrier def get_balance(self, currency: str) -> float: @@ -515,7 +613,7 @@ class Exchange: return self._api.fetch_tickers() except ccxt.NotSupported as e: raise OperationalException( - f'Exchange {self._api.name} does not support fetching tickers in batch.' + f'Exchange {self._api.name} does not support fetching tickers in batch. ' f'Message: {e}') from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( @@ -524,39 +622,28 @@ class Exchange: raise OperationalException(e) from e @retrier - def get_ticker(self, pair: str, refresh: Optional[bool] = True) -> dict: - if refresh or pair not in self._cached_ticker.keys(): - try: - if pair not in self._api.markets or not self._api.markets[pair].get('active'): - raise DependencyException(f"Pair {pair} not available") - data = self._api.fetch_ticker(pair) - try: - self._cached_ticker[pair] = { - 'bid': float(data['bid']), - 'ask': float(data['ask']), - } - except KeyError: - logger.debug("Could not cache ticker data for %s", pair) - return data - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - else: - logger.info("returning cached ticker-data for %s", pair) - return self._cached_ticker[pair] + def fetch_ticker(self, pair: str) -> dict: + try: + if pair not in self._api.markets or not self._api.markets[pair].get('active'): + raise DependencyException(f"Pair {pair} not available") + data = self._api.fetch_ticker(pair) + return data + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not load ticker due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: """ - Gets candle history using asyncio and returns the list of candles. - Handles all async doing. - Async over one pair, assuming we get `_ohlcv_candle_limit` candles per call. + Get candle history using asyncio and returns the list of candles. + Handles all async work for this. + Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call. :param pair: Pair to download - :param timeframe: Ticker Timeframe to get + :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from - :returns List of tickers + :returns List with candle (OHLCV) data """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, @@ -576,26 +663,27 @@ class Exchange: pair, timeframe, since) for since in range(since_ms, arrow.utcnow().timestamp * 1000, one_call)] - tickers = await asyncio.gather(*input_coroutines, return_exceptions=True) + results = await asyncio.gather(*input_coroutines, return_exceptions=True) - # Combine tickers + # Combine gathered results data: List = [] - for p, timeframe, ticker in tickers: + for p, timeframe, res in results: if p == pair: - data.extend(ticker) + data.extend(res) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - logger.info("downloaded %s with length %s.", pair, len(data)) + logger.info("Downloaded data for %s with length %s.", pair, len(data)) return data def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]: """ - Refresh in-memory ohlcv asynchronously and set `_klines` with the result + Refresh in-memory OHLCV asynchronously and set `_klines` with the result Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). + Only used in the dataprovider.refresh() method. :param pair_list: List of 2 element tuples containing pair, interval to refresh - :return: Returns a List of ticker-dataframes. + :return: TODO: return value is only used in the tests, get rid of it """ - logger.debug("Refreshing ohlcv data for %d pairs", len(pair_list)) + logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) input_coroutines = [] @@ -606,15 +694,15 @@ class Exchange: input_coroutines.append(self._async_get_candle_history(pair, timeframe)) else: logger.debug( - "Using cached ohlcv data for pair %s, timeframe %s ...", + "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", pair, timeframe ) - tickers = asyncio.get_event_loop().run_until_complete( + results = asyncio.get_event_loop().run_until_complete( asyncio.gather(*input_coroutines, return_exceptions=True)) # handle caching - for res in tickers: + for res in results: if isinstance(res, Exception): logger.warning("Async code raised an exception: %s", res.__class__.__name__) continue @@ -625,13 +713,14 @@ class Exchange: if ticks: self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache - self._klines[(pair, timeframe)] = parse_ticker_dataframe( + self._klines[(pair, timeframe)] = ohlcv_to_dataframe( ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - return tickers + + return results def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool: - # Calculating ticker interval in seconds + # Timeframe in seconds interval_in_sec = timeframe_to_seconds(timeframe) return not ((self._pairs_last_refresh_time.get((pair, timeframe), 0) @@ -641,11 +730,11 @@ class Exchange: async def _async_get_candle_history(self, pair: str, timeframe: str, since_ms: Optional[int] = None) -> Tuple[str, str, List]: """ - Asynchronously gets candle histories using fetch_ohlcv + Asynchronously get candle history data using fetch_ohlcv returns tuple: (pair, timeframe, ohlcv_list) """ try: - # fetch ohlcv asynchronously + # Fetch OHLCV asynchronously s = '(' + arrow.get(since_ms // 1000).isoformat() + ') ' if since_ms is not None else '' logger.debug( "Fetching pair %s, interval %s, since %s %s...", @@ -655,9 +744,9 @@ class Exchange: data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, since=since_ms) - # Because some exchange sort Tickers ASC and other DESC. - # Ex: Bittrex returns a list of tickers ASC (oldest first, newest last) - # when GDAX returns a list of tickers DESC (newest first, oldest last) + # Some exchanges sort OHLCV in ASC order and others in DESC. + # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) + # while GDAX returns the list of OHLCV in DESC order (newest first, oldest last) # Only sort if necessary to save computing time try: if data and data[0][0] > data[-1][0]: @@ -670,18 +759,20 @@ class Exchange: except ccxt.NotSupported as e: raise OperationalException( - f'Exchange {self._api.name} does not support fetching historical candlestick data.' - f'Message: {e}') from e + f'Exchange {self._api.name} does not support fetching historical ' + f'candle (OHLCV) data. Message: {e}') from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError(f'Could not load ticker history due to {e.__class__.__name__}. ' + raise TemporaryError(f'Could not fetch historical candle (OHLCV) data ' + f'for pair {pair} due to {e.__class__.__name__}. ' f'Message: {e}') from e except ccxt.BaseError as e: - raise OperationalException(f'Could not fetch ticker data. Msg: {e}') from e + raise OperationalException(f'Could not fetch historical candle (OHLCV) data ' + f'for pair {pair}. Message: {e}') from e @retrier_async async def _async_fetch_trades(self, pair: str, since: Optional[int] = None, - params: Optional[dict] = None) -> List[Dict]: + params: Optional[dict] = None) -> List[List]: """ Asyncronously gets trade history using fetch_trades. Handles exchange errors, does one call to the exchange. @@ -701,7 +792,7 @@ class Exchange: '(' + arrow.get(since // 1000).isoformat() + ') ' if since is not None else '' ) trades = await self._api_async.fetch_trades(pair, since=since, limit=1000) - return trades + return trades_dict_to_list(trades) except ccxt.NotSupported as e: raise OperationalException( f'Exchange {self._api.name} does not support fetching historical trade data.' @@ -715,7 +806,7 @@ class Exchange: async def _async_get_trade_history_id(self, pair: str, until: int, since: Optional[int] = None, - from_id: Optional[str] = None) -> Tuple[str, List[Dict]]: + from_id: Optional[str] = None) -> Tuple[str, List[List]]: """ Asyncronously gets trade history using fetch_trades use this when exchange uses id-based iteration (check `self._trades_pagination`) @@ -726,7 +817,7 @@ class Exchange: returns tuple: (pair, trades-list) """ - trades: List[Dict] = [] + trades: List[List] = [] if not from_id: # Fetch first elements using timebased method to get an ID to paginate on @@ -735,7 +826,9 @@ class Exchange: # e.g. Binance returns the "last 1000" candles within a 1h time interval # - so we will miss the first trades. t = await self._async_fetch_trades(pair, since=since) - from_id = t[-1]['id'] + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id + from_id = t[-1][1] trades.extend(t[:-1]) while True: t = await self._async_fetch_trades(pair, @@ -743,21 +836,21 @@ class Exchange: if len(t): # Skip last id since its the key for the next call trades.extend(t[:-1]) - if from_id == t[-1]['id'] or t[-1]['timestamp'] > until: + if from_id == t[-1][1] or t[-1][0] > until: logger.debug(f"Stopping because from_id did not change. " - f"Reached {t[-1]['timestamp']} > {until}") + f"Reached {t[-1][0]} > {until}") # Reached the end of the defined-download period - add last trade as well. trades.extend(t[-1:]) break - from_id = t[-1]['id'] + from_id = t[-1][1] else: break return (pair, trades) async def _async_get_trade_history_time(self, pair: str, until: int, - since: Optional[int] = None) -> Tuple[str, List]: + since: Optional[int] = None) -> Tuple[str, List[List]]: """ Asyncronously gets trade history using fetch_trades, when the exchange uses time-based iteration (check `self._trades_pagination`) @@ -767,16 +860,18 @@ class Exchange: returns tuple: (pair, trades-list) """ - trades: List[Dict] = [] + trades: List[List] = [] + # DEFAULT_TRADES_COLUMNS: 0 -> timestamp + # DEFAULT_TRADES_COLUMNS: 1 -> id while True: t = await self._async_fetch_trades(pair, since=since) if len(t): - since = t[-1]['timestamp'] + since = t[-1][1] trades.extend(t) # Reached the end of the defined-download period - if until and t[-1]['timestamp'] > until: + if until and t[-1][0] > until: logger.debug( - f"Stopping because until was reached. {t[-1]['timestamp']} > {until}") + f"Stopping because until was reached. {t[-1][0]} > {until}") break else: break @@ -786,7 +881,7 @@ class Exchange: async def _async_get_trade_history(self, pair: str, since: Optional[int] = None, until: Optional[int] = None, - from_id: Optional[str] = None) -> Tuple[str, List[Dict]]: + from_id: Optional[str] = None) -> Tuple[str, List[List]]: """ Async wrapper handling downloading trades using either time or id based methods. """ @@ -809,14 +904,14 @@ class Exchange: until: Optional[int] = None, from_id: Optional[str] = None) -> Tuple[str, List]: """ - Gets candle history using asyncio and returns the list of candles. - Handles all async doing. - Async over one pair, assuming we get `_ohlcv_candle_limit` candles per call. + Get trade history data using asyncio. + Handles all async work and returns the list of candles. + Async over one pair, assuming we get `self._ohlcv_candle_limit` candles per call. :param pair: Pair to download :param since: Timestamp in milliseconds to get history from :param until: Timestamp in milliseconds. Defaults to current timestamp if not defined. :param from_id: Download data starting with ID (if id is known) - :returns List of tickers + :returns List of trade data """ if not self.exchange_has("fetchTrades"): raise OperationalException("This exchange does not suport downloading Trades.") @@ -825,10 +920,18 @@ class Exchange: self._async_get_trade_history(pair=pair, since=since, until=until, from_id=from_id)) + def check_order_canceled_empty(self, order: Dict) -> bool: + """ + Verify if an order has been cancelled without being partially filled + :param order: Order dict as returned from get_order() + :return: True if order has been cancelled without being filled, False otherwise. + """ + return order.get('status') in ('closed', 'canceled') and order.get('filled') == 0.0 + @retrier - def cancel_order(self, order_id: str, pair: str) -> None: + def cancel_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - return + return {} try: return self._api.cancel_order(order_id, pair) @@ -841,6 +944,37 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def is_cancel_order_result_suitable(self, corder) -> bool: + if not isinstance(corder, dict): + return False + + required = ('fee', 'status', 'amount') + return all(k in corder for k in required) + + def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: + """ + Cancel order returning a result. + Creates a fake result if cancel order returns a non-usable result + and get_order does not work (certain exchanges don't return cancelled orders) + :param order_id: Orderid to cancel + :param pair: Pair corresponding to order_id + :param amount: Amount to use for fake response + :return: Result from either cancel_order if usable, or fetch_order + """ + try: + corder = self.cancel_order(order_id, pair) + if self.is_cancel_order_result_suitable(corder): + return corder + except InvalidOrderException: + logger.warning(f"Could not cancel order {order_id}.") + try: + order = self.get_order(order_id, pair) + except InvalidOrderException: + logger.warning(f"Could not fetch cancelled order {order_id}.") + order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + + return order + @retrier def get_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: @@ -914,15 +1048,15 @@ class Exchange: return matched_trades - except ccxt.NetworkError as e: + except (ccxt.NetworkError, ccxt.ExchangeError) as e: raise TemporaryError( - f'Could not get trades due to networking error. Message: {e}') from e + f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e @retrier - def get_fee(self, symbol, type='', side='', amount=1, - price=1, taker_or_maker='maker') -> float: + def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, + price: float = 1, taker_or_maker: str = 'maker') -> float: try: # validate that markets are loaded before trying to get fee if self._api.markets is None or len(self._api.markets) == 0: @@ -936,6 +1070,61 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + @staticmethod + def order_has_fee(order: Dict) -> bool: + """ + Verifies if the passed in order dict has the needed keys to extract fees, + and that these keys (currency, cost) are not empty. + :param order: Order or trade (one trade) dict + :return: True if the fee substructure contains currency and cost, false otherwise + """ + if not isinstance(order, dict): + return False + return ('fee' in order and order['fee'] is not None + and (order['fee'].keys() >= {'currency', 'cost'}) + and order['fee']['currency'] is not None + and order['fee']['cost'] is not None + ) + + def calculate_fee_rate(self, order: Dict) -> Optional[float]: + """ + Calculate fee rate if it's not given by the exchange. + :param order: Order or trade (one trade) dict + """ + if order['fee'].get('rate') is not None: + return order['fee'].get('rate') + fee_curr = order['fee']['currency'] + # Calculate fee based on order details + if fee_curr in self.get_pair_base_currency(order['symbol']): + # Base currency - divide by amount + return round( + order['fee']['cost'] / safe_value_fallback(order, order, 'filled', 'amount'), 8) + elif fee_curr in self.get_pair_quote_currency(order['symbol']): + # Quote currency - divide by cost + return round(order['fee']['cost'] / order['cost'], 8) + else: + # If Fee currency is a different currency + try: + comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) + tick = self.fetch_ticker(comb) + + fee_to_quote_rate = safe_value_fallback(tick, tick, 'last', 'ask') + return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) + except DependencyException: + return None + + def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: + """ + Extract tuple of cost, currency, rate. + Requires order_has_fee to run first! + :param order: Order or trade (one trade) dict + :return: Tuple with cost, currency, rate of the given fee dict + """ + return (order['fee']['cost'], + order['fee']['currency'], + self.calculate_fee_rate(order)) + # calculate rate ? (order['fee']['cost'] / (order['amount'] * order['price'])) + def is_exchange_bad(exchange_name: str) -> bool: return exchange_name in BAD_EXCHANGES @@ -945,22 +1134,22 @@ def get_exchange_bad_reason(exchange_name: str) -> str: return BAD_EXCHANGES.get(exchange_name, "") -def is_exchange_known_ccxt(exchange_name: str, ccxt_module=None) -> bool: +def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool: return exchange_name in ccxt_exchanges(ccxt_module) def is_exchange_officially_supported(exchange_name: str) -> bool: - return exchange_name in ['bittrex', 'binance'] + return exchange_name in ['bittrex', 'binance', 'kraken'] -def ccxt_exchanges(ccxt_module=None) -> List[str]: +def ccxt_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: """ Return the list of all exchanges known to ccxt """ return ccxt_module.exchanges if ccxt_module is not None else ccxt.exchanges -def available_exchanges(ccxt_module=None) -> List[str]: +def available_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]: """ Return exchanges available to the bot, i.e. non-bad exchanges in the ccxt list """ @@ -1020,7 +1209,8 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime: return datetime.fromtimestamp(new_timestamp, tz=timezone.utc) -def symbol_is_pair(market_symbol: str, base_currency: str = None, quote_currency: str = None): +def symbol_is_pair(market_symbol: str, base_currency: str = None, + quote_currency: str = None) -> bool: """ Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the quote currency separated by '/' character. If base_currency and/or quote_currency is passed, @@ -1033,7 +1223,7 @@ def symbol_is_pair(market_symbol: str, base_currency: str = None, quote_currency (symbol_parts[1] == quote_currency if quote_currency else len(symbol_parts[1]) > 0)) -def market_is_active(market): +def market_is_active(market: Dict) -> bool: """ Return True if the market is active. """ diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py new file mode 100644 index 000000000..75915122b --- /dev/null +++ b/freqtrade/exchange/ftx.py @@ -0,0 +1,14 @@ +""" FTX exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + +logger = logging.getLogger(__name__) + + +class Ftx(Exchange): + + _ft_has: Dict = { + "ohlcv_candle_limit": 1500, + } diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index f548489bc..243f1a6d6 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -4,7 +4,8 @@ from typing import Dict import ccxt -from freqtrade import OperationalException, TemporaryError +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange import Exchange from freqtrade.exchange.exchange import retrier @@ -15,6 +16,7 @@ class Kraken(Exchange): _params: Dict = {"trading_agreement": "agree"} _ft_has: Dict = { + "stoploss_on_exchange": True, "trades_pagination": "id", "trades_pagination_arg": "since", } @@ -48,3 +50,51 @@ class Kraken(Exchange): f'Could not get balance due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + + def stoploss_adjust(self, stop_loss: float, order: Dict) -> bool: + """ + Verify stop_loss against stoploss-order value (limit or price) + Returns True if adjustment is necessary. + """ + return order['type'] == 'stop-loss' and stop_loss > float(order['price']) + + def stoploss(self, pair: str, amount: float, stop_price: float, order_types: Dict) -> Dict: + """ + Creates a stoploss market order. + Stoploss market orders is the only stoploss type supported by kraken. + """ + + ordertype = "stop-loss" + + stop_price = self.price_to_precision(pair, stop_price) + + if self._config['dry_run']: + dry_order = self.dry_run_order( + pair, ordertype, "sell", amount, stop_price) + return dry_order + + try: + params = self._params.copy() + + amount = self.amount_to_precision(pair, amount) + + order = self._api.create_order(symbol=pair, type=ordertype, side='sell', + amount=amount, price=stop_price, params=params) + logger.info('stoploss order added for %s. ' + 'stop price: %s.', pair, stop_price) + return order + except ccxt.InsufficientFunds as e: + raise DependencyException( + f'Insufficient funds to create {ordertype} sell order on market {pair}.' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not create {ordertype} sell order on market {pair}. ' + f'Tried to create stoploss with amount {amount} at stoploss {stop_price}. ' + f'Message: {e}') from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8ae027fa2..a6d32d8fe 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -6,25 +6,28 @@ import logging import traceback from datetime import datetime from math import isclose -from os import getpid +from threading import Lock from typing import Any, Dict, List, Optional, Tuple import arrow +from cachetools import TTLCache from requests.exceptions import RequestException -from freqtrade import (DependencyException, InvalidOrderException, __version__, - constants, persistence) +from freqtrade import __version__, constants, persistence from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge +from freqtrade.exceptions import DependencyException, InvalidOrderException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_next_date +from freqtrade.misc import safe_value_fallback +from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.state import State from freqtrade.strategy.interface import IStrategy, SellType +from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) @@ -51,31 +54,32 @@ class FreqtradeBot: # Init objects self.config = config - self._heartbeat_msg = 0 + # Cache values for 1800 to avoid frequent polling of the exchange for prices + # Caching only applies to RPC methods, so prices for open trades are still + # refreshed once every iteration. + self._sell_rate_cache = TTLCache(maxsize=100, ttl=1800) + self._buy_rate_cache = TTLCache(maxsize=100, ttl=1800) - self.heartbeat_interval = self.config.get('internals', {}).get('heartbeat_interval', 60) - - self.strategy: IStrategy = StrategyResolver(self.config).strategy + self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) # Check config consistency here since strategies can set certain options validate_config_consistency(config) - self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange + self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) - persistence.init(self.config.get('db_url', None), - clean_open_orders=self.config.get('dry_run', False)) + persistence.init(self.config.get('db_url', None), clean_open_orders=self.config['dry_run']) self.wallets = Wallets(self.config, self.exchange) - self.dataprovider = DataProvider(self.config, self.exchange) + self.pairlists = PairListManager(self.exchange, self.config) + + self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass IStrategy.wallets = self.wallets - self.pairlists = PairListManager(self.exchange, self.config) - # Initializing Edge only if enabled self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None @@ -92,6 +96,18 @@ class FreqtradeBot: # the initial state of the bot. # Keep this at the end of this initialization method. self.rpc: RPCManager = RPCManager(self) + # Protect sell-logic from forcesell and viceversa + self._sell_lock = Lock() + + def notify_status(self, msg: str) -> None: + """ + Public method for users of this class (worker, etc.) to send notifications + via RPC about changes in the bot status. + """ + self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': msg + }) def cleanup(self) -> None: """ @@ -100,6 +116,9 @@ class FreqtradeBot: """ logger.info('Cleaning up modules ...') + if self.config['cancel_open_orders_on_exit']: + self.cancel_all_open_orders() + self.rpc.cleanup() persistence.cleanup() @@ -132,21 +151,29 @@ class FreqtradeBot: self.dataprovider.refresh(self._create_pair_whitelist(self.active_pair_whitelist), self.strategy.informative_pairs()) - # First process current opened trades - self.process_maybe_execute_sells(trades) + with self._sell_lock: + # Check and handle any timed out open orders + self.check_handle_timedout() + + # Protect from collisions with forcesell. + # Without this, freqtrade my try to recreate stoploss_on_exchange orders + # while selling is in process, since telegram messages arrive in an different thread. + with self._sell_lock: + # First process current opened trades (positions) + self.exit_positions(trades) # Then looking for buy opportunities - if len(trades) < self.config['max_open_trades']: - self.process_maybe_execute_buys() + if self.get_free_open_trades(): + self.enter_positions() - # Check and handle any timed out open orders - self.check_handle_timedout() Trade.session.flush() - if (self.heartbeat_interval - and (arrow.utcnow().timestamp - self._heartbeat_msg > self.heartbeat_interval)): - logger.info(f"Bot heartbeat. PID={getpid()}") - self._heartbeat_msg = arrow.utcnow().timestamp + def process_stopped(self) -> None: + """ + Close all orders that were left open + """ + if self.config['cancel_open_orders_on_exit']: + self.cancel_all_open_orders() def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]: """ @@ -162,8 +189,8 @@ class FreqtradeBot: _whitelist = self.edge.adjust(_whitelist) if trades: - # Extend active-pair whitelist with pairs from open trades - # It ensures that tickers are downloaded for open trades + # Extend active-pair whitelist with pairs of open trades + # It ensures that candle (OHLCV) data are downloaded for open trades as well _whitelist.extend([trade.pair for trade in trades if trade.pair not in _whitelist]) return _whitelist @@ -173,45 +200,102 @@ class FreqtradeBot: """ return [(pair, self.config['ticker_interval']) for pair in pairs] - def get_target_bid(self, pair: str, tick: Dict = None) -> float: + def get_free_open_trades(self): + """ + Return the number of free open trades slots or 0 if + max number of open trades reached + """ + open_trades = len(Trade.get_open_trades()) + return max(0, self.config['max_open_trades'] - open_trades) + +# +# BUY / enter positions / open trades logic and methods +# + + def enter_positions(self) -> int: + """ + Tries to execute buy orders for new trades (positions) + """ + trades_created = 0 + + whitelist = copy.deepcopy(self.active_pair_whitelist) + if not whitelist: + logger.info("Active pair whitelist is empty.") + else: + # Remove pairs for currently opened trades from the whitelist + for trade in Trade.get_open_trades(): + if trade.pair in whitelist: + whitelist.remove(trade.pair) + logger.debug('Ignoring %s in pair whitelist', trade.pair) + + if not whitelist: + logger.info("No currency pair in active pair whitelist, " + "but checking to sell open trades.") + else: + # Create entity and execute trade for each pair from whitelist + for pair in whitelist: + try: + trades_created += self.create_trade(pair) + except DependencyException as exception: + logger.warning('Unable to create trade for %s: %s', pair, exception) + + if not trades_created: + logger.debug("Found no buy signals for whitelisted currencies. " + "Trying again...") + + return trades_created + + def get_buy_rate(self, pair: str, refresh: bool) -> float: """ Calculates bid target between current ask price and last price + :param pair: Pair to get rate for + :param refresh: allow cached data :return: float: Price """ - config_bid_strategy = self.config.get('bid_strategy', {}) - if 'use_order_book' in config_bid_strategy and\ - config_bid_strategy.get('use_order_book', False): - logger.info('Getting price from order book') - order_book_top = config_bid_strategy.get('order_book_top', 1) + if not refresh: + rate = self._buy_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.info(f"Using cached buy rate for {pair}.") + return rate + + bid_strategy = self.config.get('bid_strategy', {}) + if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): + logger.info( + f"Getting price from order book {bid_strategy['price_side'].capitalize()} side." + ) + order_book_top = bid_strategy.get('order_book_top', 1) order_book = self.exchange.get_order_book(pair, order_book_top) logger.debug('order_book %s', order_book) # top 1 = index 0 - order_book_rate = order_book['bids'][order_book_top - 1][0] - logger.info('...top %s order book buy rate %0.8f', order_book_top, order_book_rate) + order_book_rate = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] + logger.info(f'...top {order_book_top} order book buy rate {order_book_rate:.8f}') used_rate = order_book_rate else: - if not tick: - logger.info('Using Last Ask / Last Price') - ticker = self.exchange.get_ticker(pair) - else: - ticker = tick - if ticker['ask'] < ticker['last']: - ticker_rate = ticker['ask'] - else: + logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") + ticker = self.exchange.fetch_ticker(pair) + ticker_rate = ticker[bid_strategy['price_side']] + if ticker['last'] and ticker_rate > ticker['last']: balance = self.config['bid_strategy']['ask_last_balance'] - ticker_rate = ticker['ask'] + balance * (ticker['last'] - ticker['ask']) + ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) used_rate = ticker_rate + self._buy_rate_cache[pair] = used_rate + return used_rate - def _get_trade_stake_amount(self, pair) -> Optional[float]: + def get_trade_stake_amount(self, pair: str) -> float: """ - Check if stake amount can be fulfilled with the available balance - for the stake currency - :return: float: Stake Amount + Calculate stake amount for the trade + :return: float: Stake amount + :raise: DependencyException if the available stake amount is too low """ + stake_amount: float + # Ensure wallets are uptodate. + self.wallets.update() + if self.edge: - return self.edge.stake_amount( + stake_amount = self.edge.stake_amount( pair, self.wallets.get_free(self.config['stake_currency']), self.wallets.get_total(self.config['stake_currency']), @@ -219,17 +303,56 @@ class FreqtradeBot: ) else: stake_amount = self.config['stake_amount'] + if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: + stake_amount = self._calculate_unlimited_stake_amount() - available_amount = self.wallets.get_free(self.config['stake_currency']) + return self._check_available_stake_amount(stake_amount) - if stake_amount == constants.UNLIMITED_STAKE_AMOUNT: - open_trades = len(Trade.get_open_trades()) - if open_trades >= self.config['max_open_trades']: - logger.warning("Can't open a new trade: max number of trades is reached") - return None - return available_amount / (self.config['max_open_trades'] - open_trades) + def _get_available_stake_amount(self) -> float: + """ + Return the total currently available balance in stake currency, + respecting tradable_balance_ratio. + Calculated as + + free amount ) * tradable_balance_ratio - + """ + val_tied_up = Trade.total_open_trades_stakes() + + # Ensure % is used from the overall balance + # Otherwise we'd risk lowering stakes with each open trade. + # (tied up + current free) * ratio) - tied up + available_amount = ((val_tied_up + self.wallets.get_free(self.config['stake_currency'])) * + self.config['tradable_balance_ratio']) - val_tied_up + return available_amount + + def _calculate_unlimited_stake_amount(self) -> float: + """ + Calculate stake amount for "unlimited" stake amount + :return: 0 if max number of trades reached, else stake_amount to use. + """ + free_open_trades = self.get_free_open_trades() + if not free_open_trades: + return 0 + + available_amount = self._get_available_stake_amount() + + return available_amount / free_open_trades + + def _check_available_stake_amount(self, stake_amount: float) -> float: + """ + Check if stake amount can be fulfilled with the available balance + for the stake currency + :return: float: Stake amount + """ + available_amount = self._get_available_stake_amount() + + if self.config['amend_last_stake_amount']: + # Remaining amount needs to be at least stake_amount * last_stake_amount_min_ratio + # Otherwise the remaining amount is too low to trade. + if available_amount > (stake_amount * self.config['last_stake_amount_min_ratio']): + stake_amount = min(stake_amount, available_amount) + else: + stake_amount = 0 - # Check if stake_amount is fulfilled if available_amount < stake_amount: raise DependencyException( f"Available balance ({available_amount} {self.config['stake_currency']}) is " @@ -273,99 +396,97 @@ class FreqtradeBot: # See also #2575 at github. return max(min_stake_amounts) / amount_reserve_percent - def create_trades(self) -> bool: + def create_trade(self, pair: str) -> bool: """ - Checks the implemented trading strategy for buy-signals, using the active pair whitelist. - If a pair triggers the buy_signal a new trade record gets created. - Checks pairs as long as the open trade count is below `max_open_trades`. - :return: True if at least one trade has been created. - """ - whitelist = copy.deepcopy(self.active_pair_whitelist) + Check the implemented trading strategy for buy signals. - if not whitelist: - logger.info("Active pair whitelist is empty.") + If the pair triggers the buy signal a new trade record gets created + and the buy-order opening the trade gets issued towards the exchange. + + :return: True if a trade has been created. + """ + logger.debug(f"create_trade for pair {pair}") + + if self.strategy.is_pair_locked(pair): + logger.info(f"Pair {pair} is currently locked.") return False - # Remove currently opened and latest pairs from whitelist - for trade in Trade.get_open_trades(): - if trade.pair in whitelist: - whitelist.remove(trade.pair) - logger.debug('Ignoring %s in pair whitelist', trade.pair) - - if not whitelist: - logger.info("No currency pair in active pair whitelist, " - "but checking to sell open trades.") + # get_free_open_trades is checked before create_trade is called + # but it is still used here to prevent opening too many trades within one iteration + if not self.get_free_open_trades(): + logger.debug(f"Can't open a new trade for {pair}: max number of trades is reached.") return False - buycount = 0 # running get_signal on historical data fetched - for _pair in whitelist: - if self.strategy.is_pair_locked(_pair): - logger.info(f"Pair {_pair} is currently locked.") - continue + (buy, sell) = self.strategy.get_signal( + pair, self.strategy.ticker_interval, + self.dataprovider.ohlcv(pair, self.strategy.ticker_interval)) - (buy, sell) = self.strategy.get_signal( - _pair, self.strategy.ticker_interval, - self.dataprovider.ohlcv(_pair, self.strategy.ticker_interval)) + if buy and not sell: + stake_amount = self.get_trade_stake_amount(pair) + if not stake_amount: + logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.") + return False - if buy and not sell and len(Trade.get_open_trades()) < self.config['max_open_trades']: - stake_amount = self._get_trade_stake_amount(_pair) - if not stake_amount: - continue + logger.info(f"Buy signal found: about create a new trade with stake_amount: " + f"{stake_amount} ...") - logger.info(f"Buy signal found: about create a new trade with stake_amount: " - f"{stake_amount} ...") + bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {}) + if ((bid_check_dom.get('enabled', False)) and + (bid_check_dom.get('bids_to_ask_delta', 0) > 0)): + if self._check_depth_of_market_buy(pair, bid_check_dom): + logger.info(f'Executing Buy for {pair}.') + return self.execute_buy(pair, stake_amount) + else: + return False - bidstrat_check_depth_of_market = self.config.get('bid_strategy', {}).\ - get('check_depth_of_market', {}) - if (bidstrat_check_depth_of_market.get('enabled', False)) and\ - (bidstrat_check_depth_of_market.get('bids_to_ask_delta', 0) > 0): - if self._check_depth_of_market_buy(_pair, bidstrat_check_depth_of_market): - buycount += self.execute_buy(_pair, stake_amount) - continue - - buycount += self.execute_buy(_pair, stake_amount) - - return buycount > 0 + logger.info(f'Executing Buy for {pair}') + return self.execute_buy(pair, stake_amount) + else: + return False def _check_depth_of_market_buy(self, pair: str, conf: Dict) -> bool: """ Checks depth of market before executing a buy """ conf_bids_to_ask_delta = conf.get('bids_to_ask_delta', 0) - logger.info('checking depth of market for %s', pair) + logger.info(f"Checking depth of market for {pair} ...") order_book = self.exchange.get_order_book(pair, 1000) order_book_data_frame = order_book_to_dataframe(order_book['bids'], order_book['asks']) order_book_bids = order_book_data_frame['b_size'].sum() order_book_asks = order_book_data_frame['a_size'].sum() bids_ask_delta = order_book_bids / order_book_asks - logger.info('bids: %s, asks: %s, delta: %s', order_book_bids, - order_book_asks, bids_ask_delta) + logger.info( + f"Bids: {order_book_bids}, Asks: {order_book_asks}, Delta: {bids_ask_delta}, " + f"Bid Price: {order_book['bids'][0][0]}, Ask Price: {order_book['asks'][0][0]}, " + f"Immediate Bid Quantity: {order_book['bids'][0][1]}, " + f"Immediate Ask Quantity: {order_book['asks'][0][1]}." + ) if bids_ask_delta >= conf_bids_to_ask_delta: + logger.info(f"Bids to asks delta for {pair} DOES satisfy condition.") return True - return False + else: + logger.info(f"Bids to asks delta for {pair} does not satisfy condition.") + return False def execute_buy(self, pair: str, stake_amount: float, price: Optional[float] = None) -> bool: """ Executes a limit buy for the given pair :param pair: pair for which we want to create a LIMIT_BUY - :return: None + :return: True if a buy order is created, false if it fails. """ - pair_s = pair.replace('_', '/') - stake_currency = self.config['stake_currency'] - fiat_currency = self.config.get('fiat_display_currency', None) time_in_force = self.strategy.order_time_in_force['buy'] if price: buy_limit_requested = price else: - # Calculate amount - buy_limit_requested = self.get_target_bid(pair) + # Calculate price + buy_limit_requested = self.get_buy_rate(pair, True) - min_stake_amount = self._get_min_pair_stake_amount(pair_s, buy_limit_requested) + min_stake_amount = self._get_min_pair_stake_amount(pair, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( - f"Can't open a new trade for {pair_s}: stake amount " + f"Can't open a new trade for {pair}: stake amount " f"is too small ({stake_amount} < {min_stake_amount})" ) return False @@ -388,7 +509,7 @@ class FreqtradeBot: if float(order['filled']) == 0: logger.warning('Buy %s order with time in force %s for %s is %s by %s.' ' zero amount is fulfilled.', - order_tif, order_type, pair_s, order_status, self.exchange.name) + order_tif, order_type, pair, order_status, self.exchange.name) return False else: # the order is partially fulfilled @@ -396,7 +517,7 @@ class FreqtradeBot: # if the order is fulfilled fully or partially logger.warning('Buy %s order with time in force %s for %s is %s by %s.' ' %s amount fulfilled out of %s (%s remaining which is canceled).', - order_tif, order_type, pair_s, order_status, self.exchange.name, + order_tif, order_type, pair, order_status, self.exchange.name, order['filled'], order['amount'], order['remaining'] ) stake_amount = order['cost'] @@ -410,17 +531,6 @@ class FreqtradeBot: amount = order['amount'] buy_limit_filled_price = order['price'] - self.rpc.send_msg({ - 'type': RPCMessageType.BUY_NOTIFICATION, - 'exchange': self.exchange.name.capitalize(), - 'pair': pair_s, - 'limit': buy_limit_filled_price, - 'order_type': order_type, - 'stake_amount': stake_amount, - 'stake_currency': stake_currency, - 'fiat_currency': fiat_currency - }) - # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') trade = Trade( @@ -448,141 +558,120 @@ class FreqtradeBot: # Updating wallets self.wallets.update() + self._notify_buy(trade, order_type) + return True - def process_maybe_execute_buys(self) -> None: + def _notify_buy(self, trade: Trade, order_type: str) -> None: """ - Tries to execute buy orders for trades in a safe way + Sends rpc notification when a buy occured. """ - try: - # Create entity and execute trade - if not self.create_trades(): - logger.debug('Found no buy signals for whitelisted currencies. Trying again...') - except DependencyException as exception: - logger.warning('Unable to create trade: %s', exception) + msg = { + 'type': RPCMessageType.BUY_NOTIFICATION, + 'exchange': self.exchange.name.capitalize(), + 'pair': trade.pair, + 'limit': trade.open_rate, + 'order_type': order_type, + 'stake_amount': trade.stake_amount, + 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), + 'amount': trade.amount, + 'open_date': trade.open_date or datetime.utcnow(), + 'current_rate': trade.open_rate_requested, + } - def process_maybe_execute_sells(self, trades: List[Any]) -> None: + # Send the message + self.rpc.send_msg(msg) + + def _notify_buy_cancel(self, trade: Trade, order_type: str) -> None: """ - Tries to execute sell orders for trades in a safe way + Sends rpc notification when a buy cancel occured. """ - result = False + current_rate = self.get_buy_rate(trade.pair, False) + + msg = { + 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, + 'exchange': self.exchange.name.capitalize(), + 'pair': trade.pair, + 'limit': trade.open_rate, + 'order_type': order_type, + 'stake_amount': trade.stake_amount, + 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), + 'amount': trade.amount, + 'open_date': trade.open_date, + 'current_rate': current_rate, + } + + # Send the message + self.rpc.send_msg(msg) + +# +# SELL / exit positions / close trades logic and methods +# + + def exit_positions(self, trades: List[Any]) -> int: + """ + Tries to execute sell orders for open trades (positions) + """ + trades_closed = 0 for trade in trades: try: - self.update_trade_state(trade) if (self.strategy.order_types.get('stoploss_on_exchange') and self.handle_stoploss_on_exchange(trade)): - result = True + trades_closed += 1 continue # Check if we can sell our current pair - if trade.open_order_id is None and self.handle_trade(trade): - result = True + if trade.open_order_id is None and trade.is_open and self.handle_trade(trade): + trades_closed += 1 except DependencyException as exception: logger.warning('Unable to sell trade: %s', exception) # Updating wallets if any trade occured - if result: + if trades_closed: self.wallets.update() - def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: + return trades_closed + + def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, + order_book_min: int = 1): """ - Get real amount for the trade - Necessary for exchanges which charge fees in base currency (e.g. binance) + Helper generator to query orderbook in loop (used for early sell-order placing) """ - if order_amount is None: - order_amount = order['amount'] - # Only run for closed orders - if trade.fee_open == 0 or order['status'] == 'open': - return order_amount - - # use fee from order-dict if possible - if ('fee' in order and order['fee'] is not None and - (order['fee'].keys() >= {'currency', 'cost'})): - if (order['fee']['currency'] is not None and - order['fee']['cost'] is not None and - trade.pair.startswith(order['fee']['currency'])): - new_amount = order_amount - order['fee']['cost'] - logger.info("Applying fee on amount for %s (from %s to %s) from Order", - trade, order['amount'], new_amount) - return new_amount - - # Fallback to Trades - trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, - trade.open_date) - - if len(trades) == 0: - logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade) - return order_amount - amount = 0 - fee_abs = 0 - for exectrade in trades: - amount += exectrade['amount'] - if ("fee" in exectrade and exectrade['fee'] is not None and - (exectrade['fee'].keys() >= {'currency', 'cost'})): - # only applies if fee is in quote currency! - if (exectrade['fee']['currency'] is not None and - exectrade['fee']['cost'] is not None and - trade.pair.startswith(exectrade['fee']['currency'])): - fee_abs += exectrade['fee']['cost'] - - if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): - logger.warning(f"Amount {amount} does not match amount {trade.amount}") - raise DependencyException("Half bought? Amounts don't match") - real_amount = amount - fee_abs - if fee_abs != 0: - logger.info(f"Applying fee on amount for {trade} " - f"(from {order_amount} to {real_amount}) from Trades") - return real_amount - - def update_trade_state(self, trade, action_order: dict = None): - """ - Checks trades with open orders and updates the amount if necessary - """ - # Get order details for actual price per unit - if trade.open_order_id: - # Update trade with order values - logger.info('Found open order for %s', trade) - try: - order = action_order or self.exchange.get_order(trade.open_order_id, trade.pair) - except InvalidOrderException as exception: - logger.warning('Unable to fetch order %s: %s', trade.open_order_id, exception) - return - # Try update amount (binance-fix) - try: - new_amount = self.get_real_amount(trade, order) - if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): - order['amount'] = new_amount - # Fee was applied, so set to 0 - trade.fee_open = 0 - trade.recalc_open_trade_price() - - except DependencyException as exception: - logger.warning("Could not update trade amount: %s", exception) - - trade.update(order) - - # Updating wallets when order is closed - if not trade.is_open: - self.wallets.update() + order_book = self.exchange.get_order_book(pair, order_book_max) + for i in range(order_book_min, order_book_max + 1): + yield order_book[side][i - 1][0] def get_sell_rate(self, pair: str, refresh: bool) -> float: """ - Get sell rate - either using get-ticker bid or first bid based on orderbook + Get sell rate - either using ticker bid or first bid based on orderbook The orderbook portion is only used for rpc messaging, which would otherwise fail - for BitMex (has no bid/ask in get_ticker) + for BitMex (has no bid/ask in fetch_ticker) or remain static in any other case since it's not updating. + :param pair: Pair to get rate for + :param refresh: allow cached data :return: Bid rate """ - config_ask_strategy = self.config.get('ask_strategy', {}) - if config_ask_strategy.get('use_order_book', False): - logger.debug('Using order book to get sell rate') + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.info(f"Using cached sell rate for {pair}.") + return rate - order_book = self.exchange.get_order_book(pair, 1) - rate = order_book['bids'][0][0] + ask_strategy = self.config.get('ask_strategy', {}) + if ask_strategy.get('use_order_book', False): + # This code is only used for notifications, selling uses the generator directly + logger.info( + f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." + ) + rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) else: - rate = self.exchange.get_ticker(pair, refresh)['bid'] + rate = self.exchange.fetch_ticker(pair)[ask_strategy['price_side']] + self._sell_rate_cache[pair] = rate return rate def handle_trade(self, trade: Trade) -> bool: @@ -600,23 +689,24 @@ class FreqtradeBot: config_ask_strategy = self.config.get('ask_strategy', {}) if (config_ask_strategy.get('use_sell_signal', True) or - config_ask_strategy.get('ignore_roi_if_buy_signal')): + config_ask_strategy.get('ignore_roi_if_buy_signal', False)): (buy, sell) = self.strategy.get_signal( trade.pair, self.strategy.ticker_interval, self.dataprovider.ohlcv(trade.pair, self.strategy.ticker_interval)) if config_ask_strategy.get('use_order_book', False): - logger.info('Using order book for selling...') + logger.debug(f'Using order book for selling {trade.pair}...') # logger.debug('Order book %s',orderBook) order_book_min = config_ask_strategy.get('order_book_min', 1) order_book_max = config_ask_strategy.get('order_book_max', 1) - order_book = self.exchange.get_order_book(trade.pair, order_book_max) - + order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s", + order_book_min=order_book_min, + order_book_max=order_book_max) for i in range(order_book_min, order_book_max + 1): - order_book_rate = order_book['asks'][i - 1][0] - logger.info(' order book asks top %s: %0.8f', i, order_book_rate) - sell_rate = order_book_rate + sell_rate = next(order_book) + logger.debug(f" order book {config_ask_strategy['price_side']} top {i}: " + f"{sell_rate:0.8f}") if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True @@ -637,13 +727,10 @@ class FreqtradeBot: Force-sells the pair (using EmergencySell reason) in case of Problems creating the order. :return: True if the order succeeded, and False in case of problems. """ - # Limit price threshold: As limit price should always be below stop-price - LIMIT_PRICE_PCT = self.strategy.order_types.get('stoploss_on_exchange_limit_ratio', 0.99) - try: - stoploss_order = self.exchange.stoploss_limit(pair=trade.pair, amount=trade.amount, - stop_price=stop_price, - rate=rate * LIMIT_PRICE_PCT) + stoploss_order = self.exchange.stoploss(pair=trade.pair, amount=trade.amount, + stop_price=stop_price, + order_types=self.strategy.order_types) trade.stoploss_order_id = str(stoploss_order['id']) return True except InvalidOrderException as e: @@ -675,8 +762,24 @@ class FreqtradeBot: except InvalidOrderException as exception: logger.warning('Unable to fetch stoploss order: %s', exception) + # We check if stoploss order is fulfilled + if stoploss_order and stoploss_order['status'] == 'closed': + trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value + self.update_trade_state(trade, stoploss_order, sl_order=True) + # Lock pair for one candle to prevent immediate rebuys + self.strategy.lock_pair(trade.pair, + timeframe_to_next_date(self.config['ticker_interval'])) + self._notify_sell(trade, "stoploss") + return True + + if trade.open_order_id or not trade.is_open: + # Trade has an open Buy or Sell order, Stoploss-handling can't happen in this case + # as the Amount on the exchange is tied up in another trade. + # The trade can be closed already (sell-order fill confirmation came in this iteration) + return False + # If buy order is fulfilled but there is no stoploss, we add a stoploss on exchange - if (not trade.open_order_id and not stoploss_order): + if (not stoploss_order): stoploss = self.edge.stoploss(pair=trade.pair) if self.edge else self.strategy.stoploss @@ -695,16 +798,6 @@ class FreqtradeBot: trade.stoploss_order_id = None logger.warning('Stoploss order was cancelled, but unable to recreate one.') - # We check if stoploss order is fulfilled - if stoploss_order and stoploss_order['status'] == 'closed': - trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value - trade.update(stoploss_order) - # Lock pair for one candle to prevent immediate rebuys - self.strategy.lock_pair(trade.pair, - timeframe_to_next_date(self.config['ticker_interval'])) - self._notify_sell(trade, "stoploss") - return True - # Finally we check if stoploss on exchange should be moved up because of trailing. if stoploss_order and self.config.get('trailing_stop', False): # if trailing stoploss is enabled we check if stoploss value has changed @@ -714,7 +807,7 @@ class FreqtradeBot: return False - def handle_trailing_stoploss_on_exchange(self, trade: Trade, order): + def handle_trailing_stoploss_on_exchange(self, trade: Trade, order: dict) -> None: """ Check to see if stoploss on exchange should be updated in case of trailing stoploss on exchange @@ -722,13 +815,12 @@ class FreqtradeBot: :param order: Current on exchange stoploss order :return: None """ - - if trade.stop_loss > float(order['info']['stopPrice']): + if self.exchange.stoploss_adjust(trade.stop_loss, order): # we check if the update is neccesary update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60) if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat: # cancelling the current stoploss on exchange first - logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s})' + logger.info('Trailing stoploss: cancelling current stoploss on exchange (id:{%s}) ' 'in order to add another one ...', order['id']) try: self.exchange.cancel_order(order['id'], trade.pair) @@ -737,10 +829,8 @@ class FreqtradeBot: f"for pair {trade.pair}") # Create new stoploss order - if self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss, - rate=trade.stop_loss): - return False - else: + if not self.create_stoploss_order(trade=trade, stop_price=trade.stop_loss, + rate=trade.stop_loss): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") @@ -750,13 +840,13 @@ class FreqtradeBot: Check and execute sell """ should_sell = self.strategy.should_sell( - trade, sell_rate, datetime.utcnow(), buy, sell, - force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 + trade, sell_rate, datetime.utcnow(), buy, sell, + force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 ) if should_sell.sell_flag: + logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}') self.execute_sell(trade, sell_rate, should_sell.sell_type) - logger.info('executed sell, reason: %s', should_sell.sell_type) return True return False @@ -786,119 +876,167 @@ class FreqtradeBot: continue order = self.exchange.get_order(trade.open_order_id, trade.pair) except (RequestException, DependencyException, InvalidOrderException): - logger.info( - 'Cannot query order for %s due to %s', - trade, - traceback.format_exc()) + logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) continue - # Check if trade is still actually open - if float(order.get('remaining', 0.0)) == 0.0: - self.wallets.update() - continue + fully_cancelled = self.update_trade_state(trade, order) - if ((order['side'] == 'buy' and order['status'] == 'canceled') - or (self._check_timed_out('buy', order))): + if (order['side'] == 'buy' and (order['status'] == 'open' or fully_cancelled) and ( + fully_cancelled + or self._check_timed_out('buy', order) + or strategy_safe_wrapper(self.strategy.check_buy_timeout, + default_retval=False)(pair=trade.pair, + trade=trade, + order=order))): + self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['TIMEOUT']) - self.handle_timedout_limit_buy(trade, order) - self.wallets.update() + elif (order['side'] == 'sell' and (order['status'] == 'open' or fully_cancelled) and ( + fully_cancelled + or self._check_timed_out('sell', order) + or strategy_safe_wrapper(self.strategy.check_sell_timeout, + default_retval=False)(pair=trade.pair, + trade=trade, + order=order))): + self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['TIMEOUT']) - elif ((order['side'] == 'sell' and order['status'] == 'canceled') - or (self._check_timed_out('sell', order))): - self.handle_timedout_limit_sell(trade, order) - self.wallets.update() - - def handle_buy_order_full_cancel(self, trade: Trade, reason: str) -> None: - """Close trade in database and send message""" - Trade.session.delete(trade) - Trade.session.flush() - logger.info('Buy order %s for %s.', reason, trade) - self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Unfilled buy order for {trade.pair} {reason}' - }) - - def handle_timedout_limit_buy(self, trade: Trade, order: Dict) -> bool: + def cancel_all_open_orders(self) -> None: """ - Buy timeout - cancel order + Cancel all orders that are currently open + :return: None + """ + + for trade in Trade.get_open_order_trades(): + try: + order = self.exchange.get_order(trade.open_order_id, trade.pair) + except (DependencyException, InvalidOrderException): + logger.info('Cannot query order for %s due to %s', trade, traceback.format_exc()) + continue + + if order['side'] == 'buy': + self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + + elif order['side'] == 'sell': + self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + + def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool: + """ + Buy cancel - cancel order :return: True if order was fully cancelled """ - reason = "cancelled due to timeout" - if order['status'] != 'canceled': - corder = self.exchange.cancel_order(trade.open_order_id, trade.pair) + was_trade_fully_canceled = False + + # Cancelled orders may have the status of 'canceled' or 'closed' + if order['status'] not in ('canceled', 'closed'): + reason = constants.CANCEL_REASON['TIMEOUT'] + corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair, + trade.amount) else: # Order was cancelled already, so we can reuse the existing dict corder = order - reason = "canceled on Exchange" + reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE'] - if corder.get('remaining', order['remaining']) == order['amount']: + logger.info('Buy order %s for %s.', reason, trade) + + # Using filled to determine the filled amount + filled_amount = safe_value_fallback(corder, order, 'filled', 'filled') + + if isclose(filled_amount, 0.0, abs_tol=constants.MATH_CLOSE_PREC): + logger.info('Buy order fully cancelled. Removing %s from database.', trade) # if trade is not partially completed, just delete the trade - self.handle_buy_order_full_cancel(trade, reason) - return True + Trade.session.delete(trade) + Trade.session.flush() + was_trade_fully_canceled = True + else: + # if trade is partially complete, edit the stake details for the trade + # and close the order + # cancel_order may not contain the full order dict, so we need to fallback + # to the order dict aquired before cancelling. + # we need to fall back to the values from order if corder does not contain these keys. + trade.amount = filled_amount + trade.stake_amount = trade.amount * trade.open_rate + self.update_trade_state(trade, corder, trade.amount) - # if trade is partially complete, edit the stake details for the trade - # and close the order - # cancel_order may not contain the full order dict, so we need to fallback - # to the order dict aquired before cancelling. - # we need to fall back to the values from order if corder does not contain these keys. - trade.amount = order['amount'] - corder.get('remaining', order['remaining']) - trade.stake_amount = trade.amount * trade.open_rate - # verify if fees were taken from amount to avoid problems during selling - try: - new_amount = self.get_real_amount(trade, corder if 'fee' in corder else order, - trade.amount) - if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): - trade.amount = new_amount - # Fee was applied, so set to 0 - trade.fee_open = 0 - trade.recalc_open_trade_price() - except DependencyException as e: - logger.warning("Could not update trade amount: %s", e) + trade.open_order_id = None + logger.info('Partial buy order timeout for %s.', trade) + self.rpc.send_msg({ + 'type': RPCMessageType.STATUS_NOTIFICATION, + 'status': f'Remaining buy order for {trade.pair} cancelled due to timeout' + }) - trade.open_order_id = None - logger.info('Partial buy order timeout for %s.', trade) - self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Remaining buy order for {trade.pair} cancelled due to timeout' - }) - return False + self.wallets.update() + self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy']) + return was_trade_fully_canceled - def handle_timedout_limit_sell(self, trade: Trade, order: Dict) -> bool: + def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str: """ - Sell timeout - cancel order and update trade - :return: True if order was fully cancelled + Sell cancel - cancel order and update trade + :return: Reason for cancel """ - if order['remaining'] == order['amount']: - # if trade is not partially completed, just cancel the trade - if order["status"] != "canceled": - reason = "due to timeout" - self.exchange.cancel_order(trade.open_order_id, trade.pair) - logger.info('Sell order timeout for %s.', trade) + # if trade is not partially completed, just cancel the order + if order['remaining'] == order['amount'] or order.get('filled') == 0.0: + if not self.exchange.check_order_canceled_empty(order): + try: + # if trade is not partially completed, just delete the order + self.exchange.cancel_order(trade.open_order_id, trade.pair) + except InvalidOrderException: + logger.exception(f"Could not cancel sell order {trade.open_order_id}") + return 'error cancelling order' + logger.info('Sell order %s for %s.', reason, trade) else: - reason = "on exchange" - logger.info('Sell order canceled on exchange for %s.', trade) + reason = constants.CANCEL_REASON['CANCELLED_ON_EXCHANGE'] + logger.info('Sell order %s for %s.', reason, trade) + trade.close_rate = None + trade.close_rate_requested = None trade.close_profit = None + trade.close_profit_abs = None trade.close_date = None trade.is_open = True trade.open_order_id = None - self.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'Unfilled sell order for {trade.pair} cancelled {reason}' - }) + else: + # TODO: figure out how to handle partially complete sell orders + reason = constants.CANCEL_REASON['PARTIALLY_FILLED'] - return True + self.wallets.update() + self._notify_sell_cancel( + trade, + order_type=self.strategy.order_types['sell'], + reason=reason + ) + return reason - # TODO: figure out how to handle partially complete sell orders - return False + def _safe_sell_amount(self, pair: str, amount: float) -> float: + """ + Get sellable amount. + Should be trade.amount - but will fall back to the available amount if necessary. + This should cover cases where get_real_amount() was not able to update the amount + for whatever reason. + :param pair: Pair we're trying to sell + :param amount: amount we expect to be available + :return: amount to sell + :raise: DependencyException: if available balance is not within 2% of the available amount. + """ + # Update wallets to ensure amounts tied up in a stoploss is now free! + self.wallets.update() + trade_base_currency = self.exchange.get_pair_base_currency(pair) + wallet_amount = self.wallets.get_free(trade_base_currency) + logger.debug(f"{pair} - Wallet: {wallet_amount} - Trade-amount: {amount}") + if wallet_amount >= amount: + return amount + elif wallet_amount > amount * 0.98: + logger.info(f"{pair} - Falling back to wallet-amount {wallet_amount} -> {amount}.") + return wallet_amount + else: + raise DependencyException( + f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}") - def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> None: + def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> bool: """ Executes a limit sell for the given trade and limit :param trade: Trade instance :param limit: limit rate for the sell order :param sellreason: Reason the sell was triggered - :return: None + :return: True if it succeeds (supported) False (not supported) """ sell_type = 'sell' if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): @@ -906,7 +1044,7 @@ class FreqtradeBot: # if stoploss is on exchange and we are on dry_run mode, # we consider the sell price stop price - if self.config.get('dry_run', False) and sell_type == 'stoploss' \ + if self.config['dry_run'] and sell_type == 'stoploss' \ and self.strategy.order_types['stoploss_on_exchange']: limit = trade.stop_loss @@ -917,15 +1055,17 @@ class FreqtradeBot: except InvalidOrderException: logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}") - ordertype = self.strategy.order_types[sell_type] + order_type = self.strategy.order_types[sell_type] if sell_reason == SellType.EMERGENCY_SELL: - # Emergencysells (default to market!) - ordertype = self.strategy.order_types.get("emergencysell", "market") + # Emergency sells (default to market!) + order_type = self.strategy.order_types.get("emergencysell", "market") + + amount = self._safe_sell_amount(trade.pair, trade.amount) # Execute sell and update trade record order = self.exchange.sell(pair=str(trade.pair), - ordertype=ordertype, - amount=trade.amount, rate=limit, + ordertype=order_type, + amount=amount, rate=limit, time_in_force=self.strategy.order_time_in_force['sell'] ) @@ -934,50 +1074,229 @@ class FreqtradeBot: trade.sell_reason = sell_reason.value # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': - trade.update(order) + self.update_trade_state(trade, order) Trade.session.flush() # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, timeframe_to_next_date(self.config['ticker_interval'])) - self._notify_sell(trade, ordertype) + self._notify_sell(trade, order_type) - def _notify_sell(self, trade: Trade, order_type: str): + return True + + def _notify_sell(self, trade: Trade, order_type: str) -> None: """ Sends rpc notification when a sell occured. """ profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - # Use cached ticker here - it was updated seconds ago. + # Use cached rates here - it was updated seconds ago. current_rate = self.get_sell_rate(trade.pair, False) - profit_percent = trade.calc_profit_ratio(profit_rate) - gain = "profit" if profit_percent > 0 else "loss" + profit_ratio = trade.calc_profit_ratio(profit_rate) + gain = "profit" if profit_ratio > 0 else "loss" msg = { 'type': RPCMessageType.SELL_NOTIFICATION, 'exchange': trade.exchange.capitalize(), 'pair': trade.pair, 'gain': gain, - 'limit': trade.close_rate_requested, + 'limit': profit_rate, 'order_type': order_type, 'amount': trade.amount, 'open_rate': trade.open_rate, 'current_rate': current_rate, 'profit_amount': profit_trade, - 'profit_percent': profit_percent, + 'profit_ratio': profit_ratio, 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, - 'close_date': trade.close_date or datetime.utcnow() + 'close_date': trade.close_date or datetime.utcnow(), + 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), } - # For regular case, when the configuration exists - if 'stake_currency' in self.config and 'fiat_display_currency' in self.config: - stake_currency = self.config['stake_currency'] - fiat_currency = self.config['fiat_display_currency'] + if 'fiat_display_currency' in self.config: msg.update({ - 'stake_currency': stake_currency, - 'fiat_currency': fiat_currency, + 'fiat_currency': self.config['fiat_display_currency'], }) # Send the message self.rpc.send_msg(msg) + + def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None: + """ + Sends rpc notification when a sell cancel occured. + """ + if trade.sell_order_status == reason: + return + else: + trade.sell_order_status = reason + + profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested + profit_trade = trade.calc_profit(rate=profit_rate) + current_rate = self.get_sell_rate(trade.pair, False) + profit_ratio = trade.calc_profit_ratio(profit_rate) + gain = "profit" if profit_ratio > 0 else "loss" + + msg = { + 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'exchange': trade.exchange.capitalize(), + 'pair': trade.pair, + 'gain': gain, + 'limit': profit_rate, + 'order_type': order_type, + 'amount': trade.amount, + 'open_rate': trade.open_rate, + 'current_rate': current_rate, + 'profit_amount': profit_trade, + 'profit_ratio': profit_ratio, + 'sell_reason': trade.sell_reason, + 'open_date': trade.open_date, + 'close_date': trade.close_date, + 'stake_currency': self.config['stake_currency'], + 'fiat_currency': self.config.get('fiat_display_currency', None), + 'reason': reason, + } + + if 'fiat_display_currency' in self.config: + msg.update({ + 'fiat_currency': self.config['fiat_display_currency'], + }) + + # Send the message + self.rpc.send_msg(msg) + +# +# Common update trade state methods +# + + def update_trade_state(self, trade: Trade, action_order: dict = None, + order_amount: float = None, sl_order: bool = False) -> bool: + """ + Checks trades with open orders and updates the amount if necessary + Handles closing both buy and sell orders. + :return: True if order has been cancelled without being filled partially, False otherwise + """ + # Get order details for actual price per unit + if trade.open_order_id: + order_id = trade.open_order_id + elif trade.stoploss_order_id and sl_order: + order_id = trade.stoploss_order_id + else: + return False + # Update trade with order values + logger.info('Found open order for %s', trade) + try: + order = action_order or self.exchange.get_order(order_id, trade.pair) + except InvalidOrderException as exception: + logger.warning('Unable to fetch order %s: %s', order_id, exception) + return False + # Try update amount (binance-fix) + try: + new_amount = self.get_real_amount(trade, order, order_amount) + if not isclose(order['amount'], new_amount, abs_tol=constants.MATH_CLOSE_PREC): + order['amount'] = new_amount + order.pop('filled', None) + trade.recalc_open_trade_price() + except DependencyException as exception: + logger.warning("Could not update trade amount: %s", exception) + + if self.exchange.check_order_canceled_empty(order): + # Trade has been cancelled on exchange + # Handling of this will happen in check_handle_timeout. + return True + trade.update(order) + + # Updating wallets when order is closed + if not trade.is_open: + self.wallets.update() + return False + + def apply_fee_conditional(self, trade: Trade, trade_base_currency: str, + amount: float, fee_abs: float) -> float: + """ + Applies the fee to amount (either from Order or from Trades). + Can eat into dust if more than the required asset is available. + """ + self.wallets.update() + if fee_abs != 0 and self.wallets.get_free(trade_base_currency) >= amount: + # Eat into dust if we own more than base currency + logger.info(f"Fee amount for {trade} was in base currency - " + f"Eating Fee {fee_abs} into dust.") + elif fee_abs != 0: + real_amount = self.exchange.amount_to_precision(trade.pair, amount - fee_abs) + logger.info(f"Applying fee on amount for {trade} " + f"(from {amount} to {real_amount}).") + return real_amount + return amount + + def get_real_amount(self, trade: Trade, order: Dict, order_amount: float = None) -> float: + """ + Detect and update trade fee. + Calls trade.update_fee() uppon correct detection. + Returns modified amount if the fee was taken from the destination currency. + Necessary for exchanges which charge fees in base currency (e.g. binance) + :return: identical (or new) amount for the trade + """ + # Init variables + if order_amount is None: + order_amount = order['amount'] + # Only run for closed orders + if trade.fee_updated(order.get('side', '')) or order['status'] == 'open': + return order_amount + + trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) + # use fee from order-dict if possible + if self.exchange.order_has_fee(order): + fee_cost, fee_currency, fee_rate = self.exchange.extract_cost_curr_rate(order) + logger.info(f"Fee for Trade {trade} [{order.get('side')}]: " + f"{fee_cost:.8g} {fee_currency} - rate: {fee_rate}") + + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) + if trade_base_currency == fee_currency: + # Apply fee to amount + return self.apply_fee_conditional(trade, trade_base_currency, + amount=order_amount, fee_abs=fee_cost) + return order_amount + return self.fee_detection_from_trades(trade, order, order_amount) + + def fee_detection_from_trades(self, trade: Trade, order: Dict, order_amount: float) -> float: + """ + fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee. + """ + trades = self.exchange.get_trades_for_order(trade.open_order_id, trade.pair, + trade.open_date) + + if len(trades) == 0: + logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade) + return order_amount + fee_currency = None + amount = 0 + fee_abs = 0.0 + fee_cost = 0.0 + trade_base_currency = self.exchange.get_pair_base_currency(trade.pair) + fee_rate_array: List[float] = [] + for exectrade in trades: + amount += exectrade['amount'] + if self.exchange.order_has_fee(exectrade): + fee_cost_, fee_currency, fee_rate_ = self.exchange.extract_cost_curr_rate(exectrade) + fee_cost += fee_cost_ + if fee_rate_ is not None: + fee_rate_array.append(fee_rate_) + # only applies if fee is in quote currency! + if trade_base_currency == fee_currency: + fee_abs += fee_cost_ + # Ensure at least one trade was found: + if fee_currency: + # fee_rate should use mean + fee_rate = sum(fee_rate_array) / float(len(fee_rate_array)) if fee_rate_array else None + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) + + if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): + logger.warning(f"Amount {amount} does not match amount {trade.amount}") + raise DependencyException("Half bought? Amounts don't match") + + if fee_abs != 0: + return self.apply_fee_conditional(trade, trade_base_currency, + amount=amount, fee_abs=fee_abs) + else: + return amount diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index 27f16ecc3..153ce8c80 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -5,7 +5,7 @@ from logging import Formatter from logging.handlers import RotatingFileHandler, SysLogHandler from typing import Any, Dict, List -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) @@ -18,13 +18,13 @@ def _set_loggers(verbosity: int = 0) -> None: """ logging.getLogger('requests').setLevel( - logging.INFO if verbosity <= 1 else logging.DEBUG + logging.INFO if verbosity <= 1 else logging.DEBUG ) logging.getLogger("urllib3").setLevel( - logging.INFO if verbosity <= 1 else logging.DEBUG + logging.INFO if verbosity <= 1 else logging.DEBUG ) logging.getLogger('ccxt.base.exchange').setLevel( - logging.INFO if verbosity <= 2 else logging.DEBUG + logging.INFO if verbosity <= 2 else logging.DEBUG ) logging.getLogger('telegram').setLevel(logging.INFO) diff --git a/freqtrade/main.py b/freqtrade/main.py index 7afaeb1a2..08bdc5e32 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -4,6 +4,7 @@ Main Freqtrade bot script. Read the documentation to know what cli arguments you need. """ +from freqtrade.exceptions import FreqtradeException, OperationalException import sys # check min. python version if sys.version_info < (3, 6): @@ -13,8 +14,7 @@ if sys.version_info < (3, 6): import logging from typing import Any, List -from freqtrade import OperationalException -from freqtrade.configuration import Arguments +from freqtrade.commands import Arguments logger = logging.getLogger('freqtrade') @@ -38,8 +38,8 @@ def main(sysargv: List[str] = None) -> None: # No subcommand was issued. raise OperationalException( "Usage of Freqtrade requires a subcommand to be specified.\n" - "To have the previous behavior (bot executing trades in live/dry-run modes, " - "depending on the value of the `dry_run` setting in the config), run freqtrade " + "To have the bot executing trades in live/dry-run modes, " + "depending on the value of the `dry_run` setting in the config, run Freqtrade " "as `freqtrade trade [options...]`.\n" "To see the full list of options available, please use " "`freqtrade --help` or `freqtrade --help`." @@ -50,7 +50,7 @@ def main(sysargv: List[str] = None) -> None: except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') return_code = 0 - except OperationalException as e: + except FreqtradeException as e: logger.error(str(e)) return_code = 2 except Exception: diff --git a/freqtrade/misc.py b/freqtrade/misc.py index bcba78cf0..ac6084eb7 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -6,6 +6,7 @@ import logging import re from datetime import datetime from pathlib import Path +from typing import Any from typing.io import IO import numpy as np @@ -40,28 +41,30 @@ def datesarray_to_datetimearray(dates: np.ndarray) -> np.ndarray: return dates.dt.to_pydatetime() -def file_dump_json(filename: Path, data, is_zip=False) -> None: +def file_dump_json(filename: Path, data: Any, is_zip: bool = False) -> None: """ Dump JSON data into a file :param filename: file to create :param data: JSON Data to save :return: """ - logger.info(f'dumping json to "{filename}"') if is_zip: if filename.suffix != '.gz': filename = filename.with_suffix('.gz') + logger.info(f'dumping json to "{filename}"') + with gzip.open(filename, 'w') as fp: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) else: + logger.info(f'dumping json to "{filename}"') with open(filename, 'w') as fp: rapidjson.dump(data, fp, default=str, number_mode=rapidjson.NM_NATIVE) logger.debug(f'done json to "{filename}"') -def json_load(datafile: IO): +def json_load(datafile: IO) -> Any: """ load data with rapidjson Use this to have a consistent experience, @@ -78,18 +81,24 @@ def file_load_json(file): gzipfile = file # Try gzip file first, otherwise regular json file. if gzipfile.is_file(): - logger.debug('Loading ticker data from file %s', gzipfile) - with gzip.open(gzipfile) as tickerdata: - pairdata = json_load(tickerdata) + logger.debug(f"Loading historical data from file {gzipfile}") + with gzip.open(gzipfile) as datafile: + pairdata = json_load(datafile) elif file.is_file(): - logger.debug('Loading ticker data from file %s', file) - with open(file) as tickerdata: - pairdata = json_load(tickerdata) + logger.debug(f"Loading historical data from file {file}") + with open(file) as datafile: + pairdata = json_load(datafile) else: return None return pairdata +def pair_to_filename(pair: str) -> str: + for ch in ['/', '-', ' ', '.', '@', '$', '+', ':']: + pair = pair.replace(ch, '_') + return pair + + def format_ms_time(date: int) -> str: """ convert MS date to readable format. @@ -125,11 +134,26 @@ def round_dict(d, n): return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()} -def plural(num, singular: str, plural: str = None) -> str: +def safe_value_fallback(dict1: dict, dict2: dict, key1: str, key2: str, default_value=None): + """ + Search a value in dict1, return this if it's not None. + Fall back to dict2 - return key2 from dict2 if it's not None. + Else falls back to None. + + """ + if key1 in dict1 and dict1[key1] is not None: + return dict1[key1] + else: + if key2 in dict2 and dict2[key2] is not None: + return dict2[key2] + return default_value + + +def plural(num: float, singular: str, plural: str = None) -> str: return singular if (num == 1 or num == -1) else plural or singular + 's' -def render_template(templatefile: str, arguments: dict = {}): +def render_template(templatefile: str, arguments: dict = {}) -> str: from jinja2 import Environment, PackageLoader, select_autoescape @@ -138,5 +162,16 @@ def render_template(templatefile: str, arguments: dict = {}): autoescape=select_autoescape(['html', 'xml']) ) template = env.get_template(templatefile) - return template.render(**arguments) + + +def render_template_with_fallback(templatefile: str, templatefallbackfile: str, + arguments: dict = {}) -> str: + """ + Use templatefile if possible, otherwise fall back to templatefallbackfile + """ + from jinja2.exceptions import TemplateNotFound + try: + return render_template(templatefile, arguments) + except TemplateNotFound: + return render_template(templatefallbackfile, arguments) diff --git a/freqtrade/optimize/__init__.py b/freqtrade/optimize/__init__.py index 1f2f588ef..e69de29bb 100644 --- a/freqtrade/optimize/__init__.py +++ b/freqtrade/optimize/__init__.py @@ -1,102 +0,0 @@ -import logging -from typing import Any, Dict - -from freqtrade import DependencyException, constants, OperationalException -from freqtrade.state import RunMode -from freqtrade.utils import setup_utils_configuration - - -logger = logging.getLogger(__name__) - - -def setup_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: - """ - Prepare the configuration for the Hyperopt module - :param args: Cli args from Arguments() - :return: Configuration - """ - config = setup_utils_configuration(args, method) - - if method == RunMode.BACKTEST: - if config['stake_amount'] == constants.UNLIMITED_STAKE_AMOUNT: - raise DependencyException('stake amount could not be "%s" for backtesting' % - constants.UNLIMITED_STAKE_AMOUNT) - - return config - - -def start_backtesting(args: Dict[str, Any]) -> None: - """ - Start Backtesting script - :param args: Cli args from Arguments() - :return: None - """ - # Import here to avoid loading backtesting module when it's not used - from freqtrade.optimize.backtesting import Backtesting - - # Initialize configuration - config = setup_configuration(args, RunMode.BACKTEST) - - logger.info('Starting freqtrade in Backtesting mode') - - # Initialize backtesting object - backtesting = Backtesting(config) - backtesting.start() - - -def start_hyperopt(args: Dict[str, Any]) -> None: - """ - Start hyperopt script - :param args: Cli args from Arguments() - :return: None - """ - # Import here to avoid loading hyperopt module when it's not used - try: - from filelock import FileLock, Timeout - from freqtrade.optimize.hyperopt import Hyperopt - except ImportError as e: - raise OperationalException( - f"{e}. Please ensure that the hyperopt dependencies are installed.") from e - # Initialize configuration - config = setup_configuration(args, RunMode.HYPEROPT) - - logger.info('Starting freqtrade in Hyperopt mode') - - lock = FileLock(Hyperopt.get_lock_filename(config)) - - try: - with lock.acquire(timeout=1): - - # Remove noisy log messages - logging.getLogger('hyperopt.tpe').setLevel(logging.WARNING) - logging.getLogger('filelock').setLevel(logging.WARNING) - - # Initialize backtesting object - hyperopt = Hyperopt(config) - hyperopt.start() - - except Timeout: - logger.info("Another running instance of freqtrade Hyperopt detected.") - logger.info("Simultaneous execution of multiple Hyperopt commands is not supported. " - "Hyperopt module is resource hungry. Please run your Hyperopt sequentially " - "or on separate machines.") - logger.info("Quitting now.") - # TODO: return False here in order to help freqtrade to exit - # with non-zero exit code... - # Same in Edge and Backtesting start() functions. - - -def start_edge(args: Dict[str, Any]) -> None: - """ - Start Edge script - :param args: Cli args from Arguments() - :return: None - """ - from freqtrade.optimize.edge_cli import EdgeCli - # Initialize configuration - config = setup_configuration(args, RunMode.EDGE) - logger.info('Starting freqtrade in Edge mode') - - # Initialize Edge object - edge_cli = EdgeCli(config) - edge_cli.start() diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 726257cdd..3bf211d99 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -6,23 +6,25 @@ This module contains the backtesting logic import logging from copy import deepcopy from datetime import datetime, timedelta -from pathlib import Path -from typing import Any, Dict, List, NamedTuple, Optional +from typing import Any, Dict, List, NamedTuple, Optional, Tuple +import arrow from pandas import DataFrame -from tabulate import tabulate -from freqtrade import OperationalException from freqtrade.configuration import (TimeRange, remove_credentials, validate_config_consistency) from freqtrade.data import history +from freqtrade.data.converter import trim_dataframe from freqtrade.data.dataprovider import DataProvider +from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds -from freqtrade.misc import file_dump_json +from freqtrade.optimize.optimize_reports import (show_backtest_results, + store_backtest_result) +from freqtrade.pairlist.pairlistmanager import PairListManager from freqtrade.persistence import Trade from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.state import RunMode -from freqtrade.strategy.interface import IStrategy, SellType +from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType logger = logging.getLogger(__name__) @@ -60,12 +62,21 @@ class Backtesting: # Reset keys for backtesting remove_credentials(self.config) self.strategylist: List[IStrategy] = [] - self.exchange = ExchangeResolver(self.config['exchange']['name'], self.config).exchange + self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) + + self.pairlists = PairListManager(self.exchange, self.config) + if 'VolumePairList' in self.pairlists.name_list: + raise OperationalException("VolumePairList not allowed for backtesting.") + + self.pairlists.refresh_pairlist() + + if len(self.pairlists.whitelist) == 0: + raise OperationalException("No pair in whitelist.") if config.get('fee'): self.fee = config['fee'] else: - self.fee = self.exchange.get_fee(symbol=self.config['exchange']['pair_whitelist'][0]) + self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) if self.config.get('runmode') != RunMode.HYPEROPT: self.dataprovider = DataProvider(self.config, self.exchange) @@ -75,17 +86,17 @@ class Backtesting: for strat in list(self.config['strategy_list']): stratconf = deepcopy(self.config) stratconf['strategy'] = strat - self.strategylist.append(StrategyResolver(stratconf).strategy) + self.strategylist.append(StrategyResolver.load_strategy(stratconf)) validate_config_consistency(stratconf) else: # No strategy list specified, only one strategy - self.strategylist.append(StrategyResolver(self.config).strategy) + self.strategylist.append(StrategyResolver.load_strategy(self.config)) validate_config_consistency(self.config) if "ticker_interval" not in self.config: - raise OperationalException("Ticker-interval needs to be set in either configuration " - "or as cli argument `--ticker-interval 5m`") + raise OperationalException("Timeframe (ticker interval) needs to be set in either " + "configuration or as cli argument `--ticker-interval 5m`") self.timeframe = str(self.config.get('ticker_interval')) self.timeframe_min = timeframe_to_minutes(self.timeframe) @@ -104,17 +115,18 @@ class Backtesting: # And the regular "stoploss" function would not apply to that case self.strategy.order_types['stoploss_on_exchange'] = False - def load_bt_data(self): + def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]: timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) data = history.load_data( - datadir=Path(self.config['datadir']), - pairs=self.config['exchange']['pair_whitelist'], + datadir=self.config['datadir'], + pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=timerange, startup_candles=self.required_startup, fail_without_data=True, + data_format=self.config.get('dataformat_ohlcv', 'json'), ) min_date, max_date = history.get_timerange(data) @@ -129,139 +141,36 @@ class Backtesting: return data, timerange - def _generate_text_table(self, data: Dict[str, Dict], results: DataFrame, - skip_nan: bool = False) -> str: + def _get_ohlcv_as_lists(self, processed: Dict) -> Dict[str, DataFrame]: """ - Generates and returns a text table for the given backtest data and the results dataframe - :return: pretty printed table with tabulate as str - """ - stake_currency = str(self.config.get('stake_currency')) - max_open_trades = self.config.get('max_open_trades') - - floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f') - tabular_data = [] - headers = ['pair', 'buy count', 'avg profit %', 'cum profit %', - 'tot profit ' + stake_currency, 'tot profit %', 'avg duration', - 'profit', 'loss'] - for pair in data: - result = results[results.pair == pair] - if skip_nan and result.profit_abs.isnull().all(): - continue - - tabular_data.append([ - pair, - len(result.index), - result.profit_percent.mean() * 100.0, - result.profit_percent.sum() * 100.0, - result.profit_abs.sum(), - result.profit_percent.sum() * 100.0 / max_open_trades, - str(timedelta( - minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', - len(result[result.profit_abs > 0]), - len(result[result.profit_abs < 0]) - ]) - - # Append Total - tabular_data.append([ - 'TOTAL', - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_percent.sum() * 100.0, - results.profit_abs.sum(), - results.profit_percent.sum() * 100.0 / max_open_trades, - str(timedelta( - minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', - len(results[results.profit_abs > 0]), - len(results[results.profit_abs < 0]) - ]) - # Ignore type as floatfmt does allow tuples but mypy does not know that - return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore - - def _generate_text_table_sell_reason(self, data: Dict[str, Dict], results: DataFrame) -> str: - """ - Generate small table outlining Backtest results - """ - tabular_data = [] - headers = ['Sell Reason', 'Count'] - for reason, count in results['sell_reason'].value_counts().iteritems(): - tabular_data.append([reason.value, count]) - return tabulate(tabular_data, headers=headers, tablefmt="pipe") - - def _generate_text_table_strategy(self, all_results: dict) -> str: - """ - Generate summary table per strategy - """ - stake_currency = str(self.config.get('stake_currency')) - max_open_trades = self.config.get('max_open_trades') - - floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f') - tabular_data = [] - headers = ['Strategy', 'buy count', 'avg profit %', 'cum profit %', - 'tot profit ' + stake_currency, 'tot profit %', 'avg duration', - 'profit', 'loss'] - for strategy, results in all_results.items(): - tabular_data.append([ - strategy, - len(results.index), - results.profit_percent.mean() * 100.0, - results.profit_percent.sum() * 100.0, - results.profit_abs.sum(), - results.profit_percent.sum() * 100.0 / max_open_trades, - str(timedelta( - minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', - len(results[results.profit_abs > 0]), - len(results[results.profit_abs < 0]) - ]) - # Ignore type as floatfmt does allow tuples but mypy does not know that - return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore - - def _store_backtest_result(self, recordfilename: Path, results: DataFrame, - strategyname: Optional[str] = None) -> None: - - records = [(t.pair, t.profit_percent, t.open_time.timestamp(), - t.close_time.timestamp(), t.open_index - 1, t.trade_duration, - t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value) - for index, t in results.iterrows()] - - if records: - if strategyname: - # Inject strategyname to filename - recordfilename = Path.joinpath( - recordfilename.parent, - f'{recordfilename.stem}-{strategyname}').with_suffix(recordfilename.suffix) - logger.info(f'Dumping backtest results to {recordfilename}') - file_dump_json(recordfilename, records) - - def _get_ticker_list(self, processed) -> Dict[str, DataFrame]: - """ - Helper function to convert a processed tickerlist into a list for performance reasons. + Helper function to convert a processed dataframes into lists for performance reasons. Used by backtest() - so keep this optimized for performance. """ headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high'] - ticker: Dict = {} - # Create ticker dict + data: Dict = {} + # Create dict with data for pair, pair_data in processed.items(): pair_data.loc[:, 'buy'] = 0 # cleanup from previous run pair_data.loc[:, 'sell'] = 0 # cleanup from previous run - ticker_data = self.strategy.advise_sell( + df_analyzed = self.strategy.advise_sell( self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy() - # to avoid using data from future, we buy/sell with signal from previous candle - ticker_data.loc[:, 'buy'] = ticker_data['buy'].shift(1) - ticker_data.loc[:, 'sell'] = ticker_data['sell'].shift(1) + # To avoid using data from future, we use buy/sell signals shifted + # from the previous candle + df_analyzed.loc[:, 'buy'] = df_analyzed.loc[:, 'buy'].shift(1) + df_analyzed.loc[:, 'sell'] = df_analyzed.loc[:, 'sell'].shift(1) - ticker_data.drop(ticker_data.head(1).index, inplace=True) + df_analyzed.drop(df_analyzed.head(1).index, inplace=True) # Convert from Pandas to list for performance reasons # (Looping Pandas is slow.) - ticker[pair] = [x for x in ticker_data.itertuples()] - return ticker + data[pair] = [x for x in df_analyzed.itertuples()] + return data - def _get_close_rate(self, sell_row, trade: Trade, sell, trade_dur) -> float: + def _get_close_rate(self, sell_row, trade: Trade, sell: SellCheckTuple, + trade_dur: int) -> float: """ Get close rate for backtesting result """ @@ -302,7 +211,7 @@ class Backtesting: def _get_sell_trade_entry( self, pair: str, buy_row: DataFrame, - partial_ticker: List, trade_count_lock: Dict, + partial_ohlcv: List, trade_count_lock: Dict, stake_amount: float, max_open_trades: int) -> Optional[BacktestResult]: trade = Trade( @@ -317,7 +226,7 @@ class Backtesting: ) logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.") # calculate win/lose forwards from buy point - for sell_row in partial_ticker: + for sell_row in partial_ohlcv: if max_open_trades > 0: # Increase trade_count_lock for every iteration trade_count_lock[sell_row.date] = trade_count_lock.get(sell_row.date, 0) + 1 @@ -341,9 +250,9 @@ class Backtesting: close_rate=closerate, sell_reason=sell.sell_type ) - if partial_ticker: + if partial_ohlcv: # no sell condition found - trade stil open at end of backtest period - sell_row = partial_ticker[-1] + sell_row = partial_ohlcv[-1] bt_res = BacktestResult(pair=pair, profit_percent=trade.calc_profit_ratio(rate=sell_row.open), profit_abs=trade.calc_profit(rate=sell_row.open), @@ -365,35 +274,34 @@ class Backtesting: return bt_res return None - def backtest(self, args: Dict) -> DataFrame: + def backtest(self, processed: Dict, stake_amount: float, + start_date: arrow.Arrow, end_date: arrow.Arrow, + max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame: """ - Implements backtesting functionality + Implement backtesting functionality NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized. Of course try to not have ugly code. By some accessor are sometime slower than functions. - Avoid, logging on this method + Avoid extensive logging in this method and functions it calls. - :param args: a dict containing: - stake_amount: btc amount to use for each trade - processed: a processed dictionary with format {pair, data} - max_open_trades: maximum number of concurrent trades (default: 0, disabled) - position_stacking: do we allow position stacking? (default: False) - :return: DataFrame + :param processed: a processed dictionary with format {pair, data} + :param stake_amount: amount to use for each trade + :param start_date: backtesting timerange start datetime + :param end_date: backtesting timerange end datetime + :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited + :param position_stacking: do we allow position stacking? + :return: DataFrame with trades (results of backtesting) """ - # Arguments are long and noisy, so this is commented out. - # Uncomment if you need to debug the backtest() method. -# logger.debug(f"Start backtest, args: {args}") - processed = args['processed'] - stake_amount = args['stake_amount'] - max_open_trades = args.get('max_open_trades', 0) - position_stacking = args.get('position_stacking', False) - start_date = args['start_date'] - end_date = args['end_date'] + logger.debug(f"Run backtest, stake_amount: {stake_amount}, " + f"start_date: {start_date}, end_date: {end_date}, " + f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}" + ) trades = [] trade_count_lock: Dict = {} - # Dict of ticker-lists for performance (looping lists is a lot faster than dataframes) - ticker: Dict = self._get_ticker_list(processed) + # Use dict of lists with data for performance + # (looping lists is a lot faster than pandas DataFrames) + data: Dict = self._get_ohlcv_as_lists(processed) lock_pair_until: Dict = {} # Indexes per pair, so some pairs are allowed to have a missing start. @@ -403,12 +311,12 @@ class Backtesting: # Loop timerange and get candle for each pair at that point in time while tmp < end_date: - for i, pair in enumerate(ticker): + for i, pair in enumerate(data): if pair not in indexes: indexes[pair] = 0 try: - row = ticker[pair][indexes[pair]] + row = data[pair][indexes[pair]] except IndexError: # missing Data for one pair at the end. # Warnings for this are shown during data loading @@ -436,7 +344,7 @@ class Backtesting: # since indexes has been incremented before, we need to go one step back to # also check the buying candle for sell conditions. - trade_entry = self._get_sell_trade_entry(pair, row, ticker[pair][indexes[pair]-1:], + trade_entry = self._get_sell_trade_entry(pair, row, data[pair][indexes[pair]-1:], trade_count_lock, stake_amount, max_open_trades) @@ -455,18 +363,21 @@ class Backtesting: def start(self) -> None: """ - Run a backtesting end-to-end + Run backtesting end-to-end :return: None """ data: Dict[str, Any] = {} + logger.info('Using stake_currency: %s ...', self.config['stake_currency']) logger.info('Using stake_amount: %s ...', self.config['stake_amount']) + # Use max_open_trades in backtesting, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): max_open_trades = self.config['max_open_trades'] else: logger.info('Ignoring max_open_trades (--disable-max-market-positions was used) ...') max_open_trades = 0 + position_stacking = self.config.get('position_stacking', False) data, timerange = self.load_bt_data() @@ -476,11 +387,11 @@ class Backtesting: self._set_strategy(strat) # need to reprocess data every time to populate signals - preprocessed = self.strategy.tickerdata_to_dataframe(data) + preprocessed = self.strategy.ohlcvdata_to_dataframe(data) # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): - preprocessed[pair] = history.trim_dataframe(df, timerange) + preprocessed[pair] = trim_dataframe(df, timerange) min_date, max_date = history.get_timerange(preprocessed) logger.info( @@ -489,34 +400,15 @@ class Backtesting: ) # Execute backtest and print results all_results[self.strategy.get_strategy_name()] = self.backtest( - { - 'stake_amount': self.config.get('stake_amount'), - 'processed': preprocessed, - 'max_open_trades': max_open_trades, - 'position_stacking': self.config.get('position_stacking', False), - 'start_date': min_date, - 'end_date': max_date, - } + processed=preprocessed, + stake_amount=self.config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=max_open_trades, + position_stacking=position_stacking, ) - for strategy, results in all_results.items(): - - if self.config.get('export', False): - self._store_backtest_result(Path(self.config['exportfilename']), results, - strategy if len(self.strategylist) > 1 else None) - - print(f"Result for strategy {strategy}") - print(' BACKTESTING REPORT '.center(133, '=')) - print(self._generate_text_table(data, results)) - - print(' SELL REASON STATS '.center(133, '=')) - print(self._generate_text_table_sell_reason(data, results)) - - print(' LEFT OPEN TRADES REPORT '.center(133, '=')) - print(self._generate_text_table(data, results.loc[results.open_at_end], True)) - print() - if len(all_results) > 1: - # Print Strategy summary table - print(' Strategy Summary '.center(133, '=')) - print(self._generate_text_table_strategy(all_results)) - print('\nFor more details, please look at the detail tables above') + if self.config.get('export', False): + store_backtest_result(self.config['exportfilename'], all_results) + # Show backtest results + show_backtest_results(self.config, data, all_results) diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index a667ebb92..be19688d8 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -6,14 +6,12 @@ This module contains the edge backtesting interface import logging from typing import Any, Dict -from tabulate import tabulate - from freqtrade import constants from freqtrade.configuration import (TimeRange, remove_credentials, validate_config_consistency) from freqtrade.edge import Edge -from freqtrade.exchange import Exchange -from freqtrade.resolvers import StrategyResolver +from freqtrade.optimize.optimize_reports import generate_edge_table +from freqtrade.resolvers import ExchangeResolver, StrategyResolver logger = logging.getLogger(__name__) @@ -33,8 +31,8 @@ class EdgeCli: # Reset keys for edge remove_credentials(self.config) self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT - self.exchange = Exchange(self.config) - self.strategy = StrategyResolver(self.config).strategy + self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) + self.strategy = StrategyResolver.load_strategy(self.config) validate_config_consistency(self.config) @@ -42,38 +40,11 @@ class EdgeCli: # Set refresh_pairs to false for edge-cli (it must be true for edge) self.edge._refresh_pairs = False - self.timerange = TimeRange.parse_timerange(None if self.config.get( + self.edge._timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) - self.edge._timerange = self.timerange - - def _generate_edge_table(self, results: dict) -> str: - - floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', '.d') - tabular_data = [] - headers = ['pair', 'stoploss', 'win rate', 'risk reward ratio', - 'required risk reward', 'expectancy', 'total number of trades', - 'average duration (min)'] - - for result in results.items(): - if result[1].nb_trades > 0: - tabular_data.append([ - result[0], - result[1].stoploss, - result[1].winrate, - result[1].risk_reward_ratio, - result[1].required_risk_reward, - result[1].expectancy, - result[1].nb_trades, - round(result[1].avg_trade_duration) - ]) - - # Ignore type as floatfmt does allow tuples but mypy does not know that - return tabulate(tabular_data, headers=headers, - floatfmt=floatfmt, tablefmt="pipe") # type: ignore - def start(self) -> None: result = self.edge.calculate() if result: print('') # blank line for readability - print(self._generate_edge_table(self.edge._cached_pairs)) + print(generate_edge_table(self.edge._cached_pairs)) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 521a4d790..3a28de785 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -7,8 +7,8 @@ This module contains the hyperopt logic import locale import logging import random -import sys import warnings +from math import ceil from collections import OrderedDict from operator import itemgetter from pathlib import Path @@ -17,18 +17,22 @@ from typing import Any, Dict, List, Optional import rapidjson from colorama import Fore, Style -from colorama import init as colorama_init from joblib import (Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects) -from pandas import DataFrame +from pandas import DataFrame, json_normalize, isna +import progressbar +import tabulate +from os import path +import io -from freqtrade import OperationalException -from freqtrade.data.history import get_timerange, trim_dataframe +from freqtrade.data.converter import trim_dataframe +from freqtrade.data.history import get_timerange +from freqtrade.exceptions import OperationalException from freqtrade.misc import plural, round_dict from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules -from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F4 -from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F4 +from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 +from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 from freqtrade.resolvers.hyperopt_resolver import (HyperOptLossResolver, HyperOptResolver) @@ -38,15 +42,16 @@ with warnings.catch_warnings(): from skopt import Optimizer from skopt.space import Dimension - +progressbar.streams.wrap_stderr() +progressbar.streams.wrap_stdout() logger = logging.getLogger(__name__) INITIAL_POINTS = 30 -# Keep no more than 2*SKOPT_MODELS_MAX_NUM models -# in the skopt models list -SKOPT_MODELS_MAX_NUM = 10 +# Keep no more than SKOPT_MODEL_QUEUE_SIZE models +# in the skopt model queue, to optimize memory consumption +SKOPT_MODEL_QUEUE_SIZE = 10 MAX_LOSS = 100000 # just a big enough number to be bad result in loss optimization @@ -59,20 +64,21 @@ class Hyperopt: hyperopt = Hyperopt(config) hyperopt.start() """ + def __init__(self, config: Dict[str, Any]) -> None: self.config = config self.backtesting = Backtesting(self.config) - self.custom_hyperopt = HyperOptResolver(self.config).hyperopt + self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) - self.custom_hyperoptloss = HyperOptLossResolver(self.config).hyperoptloss + self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config) self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function - self.trials_file = (self.config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - self.tickerdata_pickle = (self.config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_tickerdata.pkl') + self.results_file = (self.config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_results.pickle') + self.data_pickle_file = (self.config['user_data_dir'] / + 'hyperopt_results' / 'hyperopt_tickerdata.pkl') self.total_epochs = config.get('epochs', 0) self.current_best_loss = 100 @@ -82,21 +88,21 @@ class Hyperopt: else: logger.info("Continuing on previous hyperopt results.") - self.num_trials_saved = 0 + self.num_epochs_saved = 0 # Previous evaluations - self.trials: List = [] + self.epochs: List = [] # Populate functions here (hasattr is slow so should not be run during "regular" operations) if hasattr(self.custom_hyperopt, 'populate_indicators'): self.backtesting.strategy.advise_indicators = \ - self.custom_hyperopt.populate_indicators # type: ignore + self.custom_hyperopt.populate_indicators # type: ignore if hasattr(self.custom_hyperopt, 'populate_buy_trend'): self.backtesting.strategy.advise_buy = \ - self.custom_hyperopt.populate_buy_trend # type: ignore + self.custom_hyperopt.populate_buy_trend # type: ignore if hasattr(self.custom_hyperopt, 'populate_sell_trend'): self.backtesting.strategy.advise_sell = \ - self.custom_hyperopt.populate_sell_trend # type: ignore + self.custom_hyperopt.populate_sell_trend # type: ignore # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): @@ -113,19 +119,20 @@ class Hyperopt: self.config['ask_strategy']['use_sell_signal'] = True self.print_all = self.config.get('print_all', False) + self.hyperopt_table_header = 0 self.print_colorized = self.config.get('print_colorized', False) self.print_json = self.config.get('print_json', False) @staticmethod - def get_lock_filename(config) -> str: + def get_lock_filename(config: Dict[str, Any]) -> str: return str(config['user_data_dir'] / 'hyperopt.lock') - def clean_hyperopt(self): + def clean_hyperopt(self) -> None: """ Remove hyperopt pickle files to restart hyperopt. """ - for f in [self.tickerdata_pickle, self.trials_file]: + for f in [self.data_pickle_file, self.results_file]: p = Path(f) if p.is_file(): logger.info(f"Removing `{p}`.") @@ -144,27 +151,26 @@ class Hyperopt: # and the values are taken from the list of parameters. return {d.name: v for d, v in zip(dimensions, raw_params)} - def save_trials(self, final: bool = False) -> None: + def _save_results(self) -> None: """ - Save hyperopt trials to file + Save hyperopt results to file """ - num_trials = len(self.trials) - if num_trials > self.num_trials_saved: - logger.info(f"Saving {num_trials} {plural(num_trials, 'epoch')}.") - dump(self.trials, self.trials_file) - self.num_trials_saved = num_trials - if final: - logger.info(f"{num_trials} {plural(num_trials, 'epoch')} " - f"saved to '{self.trials_file}'.") + num_epochs = len(self.epochs) + if num_epochs > self.num_epochs_saved: + logger.debug(f"Saving {num_epochs} {plural(num_epochs, 'epoch')}.") + dump(self.epochs, self.results_file) + self.num_epochs_saved = num_epochs + logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " + f"saved to '{self.results_file}'.") @staticmethod - def _read_trials(trials_file) -> List: + def _read_results(results_file: Path) -> List: """ - Read hyperopt trials file + Read hyperopt results from file """ - logger.info("Reading Trials from '%s'", trials_file) - trials = load(trials_file) - return trials + logger.info("Reading epochs from '%s'", results_file) + data = load(results_file) + return data def _get_params_details(self, params: Dict) -> Dict: """ @@ -189,7 +195,7 @@ class Hyperopt: return result @staticmethod - def print_epoch_details(results, total_epochs, print_json: bool, + def print_epoch_details(results, total_epochs: int, print_json: bool, no_header: bool = False, header_str: str = None) -> None: """ Display details of the hyperopt result @@ -218,7 +224,7 @@ class Hyperopt: Hyperopt._params_pretty_print(params, 'trailing', "Trailing stop:") @staticmethod - def _params_update_for_json(result_dict, params, space: str): + def _params_update_for_json(result_dict, params, space: str) -> None: if space in params: space_params = Hyperopt._space_params(params, space) if space in ['buy', 'sell']: @@ -235,7 +241,7 @@ class Hyperopt: result_dict.update(space_params) @staticmethod - def _params_pretty_print(params, space: str, header: str): + def _params_pretty_print(params, space: str, header: str) -> None: if space in params: space_params = Hyperopt._space_params(params, space, 5) if space == 'stoploss': @@ -251,7 +257,7 @@ class Hyperopt: return round_dict(d, r) if r else d @staticmethod - def is_best_loss(results, current_best_loss) -> bool: + def is_best_loss(results, current_best_loss: float) -> bool: return results['loss'] < current_best_loss def print_results(self, results) -> None: @@ -259,33 +265,16 @@ class Hyperopt: Log results if it is better than any previous evaluation """ is_best = results['is_best'] - if not self.print_all: - # Print '\n' after each 100th epoch to separate dots from the log messages. - # Otherwise output is messy on a terminal. - print('.', end='' if results['current_epoch'] % 100 != 0 else None) # type: ignore - sys.stdout.flush() if self.print_all or is_best: - if not self.print_all: - # Separate the results explanation string from dots - print("\n") - self.print_results_explanation(results, self.total_epochs, self.print_all, - self.print_colorized) - - @staticmethod - def print_results_explanation(results, total_epochs, highlight_best: bool, - print_colorized: bool) -> None: - """ - Log results explanation string - """ - explanation_str = Hyperopt._format_explanation_string(results, total_epochs) - # Colorize output - if print_colorized: - if results['total_profit'] > 0: - explanation_str = Fore.GREEN + explanation_str - if highlight_best and results['is_best']: - explanation_str = Style.BRIGHT + explanation_str - print(explanation_str) + print( + self.get_result_table( + self.config, results, self.total_epochs, + self.print_all, self.print_colorized, + self.hyperopt_table_header + ) + ) + self.hyperopt_table_header = 2 @staticmethod def _format_explanation_string(results, total_epochs) -> str: @@ -294,6 +283,151 @@ class Hyperopt: f"{results['results_explanation']} " + f"Objective: {results['loss']:.5f}") + @staticmethod + def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, + print_colorized: bool, remove_header: int) -> str: + """ + Log result table + """ + if not results: + return '' + + tabulate.PRESERVE_WHITESPACE = True + + trials = json_normalize(results, max_level=1) + trials['Best'] = '' + trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', + 'results_metrics.avg_profit', 'results_metrics.total_profit', + 'results_metrics.profit', 'results_metrics.duration', + 'loss', 'is_initial_point', 'is_best']] + trials.columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', + 'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best'] + trials['is_profit'] = False + trials.loc[trials['is_initial_point'], 'Best'] = '* ' + trials.loc[trials['is_best'], 'Best'] = 'Best' + trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best' + trials.loc[trials['Total profit'] > 0, 'is_profit'] = True + trials['Trades'] = trials['Trades'].astype(str) + + trials['Epoch'] = trials['Epoch'].apply( + lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs) + ) + trials['Avg profit'] = trials['Avg profit'].apply( + lambda x: '{:,.2f}%'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ') + ) + trials['Avg duration'] = trials['Avg duration'].apply( + lambda x: '{:,.1f} m'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ') + ) + trials['Objective'] = trials['Objective'].apply( + lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ') + ) + + trials['Profit'] = trials.apply( + lambda x: '{:,.8f} {} {}'.format( + x['Total profit'], config['stake_currency'], + '({:,.2f}%)'.format(x['Profit']).rjust(10, ' ') + ).rjust(25+len(config['stake_currency'])) + if x['Total profit'] != 0.0 else '--'.rjust(25+len(config['stake_currency'])), + axis=1 + ) + trials = trials.drop(columns=['Total profit']) + + if print_colorized: + for i in range(len(trials)): + if trials.loc[i]['is_profit']: + for j in range(len(trials.loc[i])-3): + trials.iat[i, j] = "{}{}{}".format(Fore.GREEN, + str(trials.loc[i][j]), Fore.RESET) + if trials.loc[i]['is_best'] and highlight_best: + for j in range(len(trials.loc[i])-3): + trials.iat[i, j] = "{}{}{}".format(Style.BRIGHT, + str(trials.loc[i][j]), Style.RESET_ALL) + + trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) + if remove_header > 0: + table = tabulate.tabulate( + trials.to_dict(orient='list'), tablefmt='orgtbl', + headers='keys', stralign="right" + ) + + table = table.split("\n", remove_header)[remove_header] + elif remove_header < 0: + table = tabulate.tabulate( + trials.to_dict(orient='list'), tablefmt='psql', + headers='keys', stralign="right" + ) + table = "\n".join(table.split("\n")[0:remove_header]) + else: + table = tabulate.tabulate( + trials.to_dict(orient='list'), tablefmt='psql', + headers='keys', stralign="right" + ) + return table + + @staticmethod + def export_csv_file(config: dict, results: list, total_epochs: int, highlight_best: bool, + csv_file: str) -> None: + """ + Log result to csv-file + """ + if not results: + return + + # Verification for overwrite + if path.isfile(csv_file): + logger.error(f"CSV file already exists: {csv_file}") + return + + try: + io.open(csv_file, 'w+').close() + except IOError: + logger.error(f"Failed to create CSV file: {csv_file}") + return + + trials = json_normalize(results, max_level=1) + trials['Best'] = '' + trials['Stake currency'] = config['stake_currency'] + + base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count', + 'results_metrics.avg_profit', 'results_metrics.total_profit', + 'Stake currency', 'results_metrics.profit', 'results_metrics.duration', + 'loss', 'is_initial_point', 'is_best'] + param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()] + trials = trials[base_metrics + param_metrics] + + base_columns = ['Best', 'Epoch', 'Trades', 'Avg profit', 'Total profit', 'Stake currency', + 'Profit', 'Avg duration', 'Objective', 'is_initial_point', 'is_best'] + param_columns = list(results[0]['params_dict'].keys()) + trials.columns = base_columns + param_columns + + trials['is_profit'] = False + trials.loc[trials['is_initial_point'], 'Best'] = '*' + trials.loc[trials['is_best'], 'Best'] = 'Best' + trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best' + trials.loc[trials['Total profit'] > 0, 'is_profit'] = True + trials['Epoch'] = trials['Epoch'].astype(str) + trials['Trades'] = trials['Trades'].astype(str) + + trials['Total profit'] = trials['Total profit'].apply( + lambda x: '{:,.8f}'.format(x) if x != 0.0 else "" + ) + trials['Profit'] = trials['Profit'].apply( + lambda x: '{:,.2f}'.format(x) if not isna(x) else "" + ) + trials['Avg profit'] = trials['Avg profit'].apply( + lambda x: '{:,.2f}%'.format(x) if not isna(x) else "" + ) + trials['Avg duration'] = trials['Avg duration'].apply( + lambda x: '{:,.1f} m'.format(x) if not isna(x) else "" + ) + trials['Objective'] = trials['Objective'].apply( + lambda x: '{:,.5f}'.format(x) if x != 100000 else "" + ) + + trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit']) + trials.to_csv(csv_file, index=False, header=True, mode='w', encoding='UTF-8') + logger.info(f"CSV file created: {csv_file}") + def has_space(self, space: str) -> bool: """ Tell if the space value is contained in the configuration @@ -345,15 +479,15 @@ class Hyperopt: if self.has_space('roi'): self.backtesting.strategy.minimal_roi = \ - self.custom_hyperopt.generate_roi_table(params_dict) + self.custom_hyperopt.generate_roi_table(params_dict) if self.has_space('buy'): self.backtesting.strategy.advise_buy = \ - self.custom_hyperopt.buy_strategy_generator(params_dict) + self.custom_hyperopt.buy_strategy_generator(params_dict) if self.has_space('sell'): self.backtesting.strategy.advise_sell = \ - self.custom_hyperopt.sell_strategy_generator(params_dict) + self.custom_hyperopt.sell_strategy_generator(params_dict) if self.has_space('stoploss'): self.backtesting.strategy.stoploss = params_dict['stoploss'] @@ -367,19 +501,17 @@ class Hyperopt: self.backtesting.strategy.trailing_only_offset_is_reached = \ d['trailing_only_offset_is_reached'] - processed = load(self.tickerdata_pickle) + processed = load(self.data_pickle_file) min_date, max_date = get_timerange(processed) backtesting_results = self.backtesting.backtest( - { - 'stake_amount': self.config['stake_amount'], - 'processed': processed, - 'max_open_trades': self.max_open_trades, - 'position_stacking': self.position_stacking, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=self.config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=self.max_open_trades, + position_stacking=self.position_stacking, ) return self._get_results_dict(backtesting_results, min_date, max_date, params_dict, params_details) @@ -438,43 +570,28 @@ class Hyperopt: n_initial_points=INITIAL_POINTS, acq_optimizer_kwargs={'n_jobs': cpu_count}, random_state=self.random_state, + model_queue_size=SKOPT_MODEL_QUEUE_SIZE, ) - def fix_optimizer_models_list(self): - """ - WORKAROUND: Since skopt is not actively supported, this resolves problems with skopt - memory usage, see also: https://github.com/scikit-optimize/scikit-optimize/pull/746 - - This may cease working when skopt updates if implementation of this intrinsic - part changes. - """ - n = len(self.opt.models) - SKOPT_MODELS_MAX_NUM - # Keep no more than 2*SKOPT_MODELS_MAX_NUM models in the skopt models list, - # remove the old ones. These are actually of no use, the current model - # from the estimator is the only one used in the skopt optimizer. - # Freqtrade code also does not inspect details of the models. - if n >= SKOPT_MODELS_MAX_NUM: - logger.debug(f"Fixing skopt models list, removing {n} old items...") - del self.opt.models[0:n] - def run_optimizer_parallel(self, parallel, asked, i) -> List: return parallel(delayed( wrap_non_picklable_objects(self.generate_optimizer))(v, i) for v in asked) @staticmethod - def load_previous_results(trials_file) -> List: + def load_previous_results(results_file: Path) -> List: """ Load data for epochs from the file if we have one """ - trials: List = [] - if trials_file.is_file() and trials_file.stat().st_size > 0: - trials = Hyperopt._read_trials(trials_file) - if trials[0].get('is_best') is None: + epochs: List = [] + if results_file.is_file() and results_file.stat().st_size > 0: + epochs = Hyperopt._read_results(results_file) + # Detection of some old format, without 'is_best' field saved + if epochs[0].get('is_best') is None: raise OperationalException( - "The file with Hyperopt results is incompatible with this version " - "of Freqtrade and cannot be loaded.") - logger.info(f"Loaded {len(trials)} previous evaluations from disk.") - return trials + "The file with Hyperopt results is incompatible with this version " + "of Freqtrade and cannot be loaded.") + logger.info(f"Loaded {len(epochs)} previous evaluations from disk.") + return epochs def _set_random_state(self, random_state: Optional[int]) -> int: return random_state or random.randint(1, 2**16 - 1) @@ -482,10 +599,10 @@ class Hyperopt: def start(self) -> None: self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None)) logger.info(f"Using optimizer random state: {self.random_state}") - + self.hyperopt_table_header = -1 data, timerange = self.backtesting.load_bt_data() - preprocessed = self.backtesting.strategy.tickerdata_to_dataframe(data) + preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data) # Trim startup period from analyzed dataframe for pair, df in preprocessed.items(): @@ -496,12 +613,13 @@ class Hyperopt: 'Hyperopting with data from %s up to %s (%s days)..', min_date.isoformat(), max_date.isoformat(), (max_date - min_date).days ) - dump(preprocessed, self.tickerdata_pickle) + dump(preprocessed, self.data_pickle_file) # We don't need exchange instance anymore while running hyperopt self.backtesting.exchange = None # type: ignore + self.backtesting.pairlists = None # type: ignore - self.trials = self.load_previous_results(self.trials_file) + self.epochs = self.load_previous_results(self.results_file) cpus = cpu_count() logger.info(f"Found {cpus} CPU cores. Let's make them scream!") @@ -510,52 +628,85 @@ class Hyperopt: self.dimensions: List[Dimension] = self.hyperopt_space() self.opt = self.get_optimizer(self.dimensions, config_jobs) - - if self.print_colorized: - colorama_init(autoreset=True) - try: with Parallel(n_jobs=config_jobs) as parallel: jobs = parallel._effective_n_jobs() logger.info(f'Effective number of parallel workers used: {jobs}') - EVALS = max(self.total_epochs // jobs, 1) - for i in range(EVALS): - asked = self.opt.ask(n_points=jobs) - f_val = self.run_optimizer_parallel(parallel, asked, i) - self.opt.tell(asked, [v['loss'] for v in f_val]) - self.fix_optimizer_models_list() - for j in range(jobs): - # Use human-friendly indexes here (starting from 1) - current = i * jobs + j + 1 - val = f_val[j] - val['current_epoch'] = current - val['is_initial_point'] = current <= INITIAL_POINTS - logger.debug(f"Optimizer epoch evaluated: {val}") - is_best = self.is_best_loss(val, self.current_best_loss) - # This value is assigned here and not in the optimization method - # to keep proper order in the list of results. That's because - # evaluations can take different time. Here they are aligned in the - # order they will be shown to the user. - val['is_best'] = is_best + # Define progressbar + if self.print_colorized: + widgets = [ + ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), + ' (', progressbar.Percentage(), ')] ', + progressbar.Bar(marker=progressbar.AnimatedMarker( + fill='\N{FULL BLOCK}', + fill_wrap=Fore.GREEN + '{}' + Fore.RESET, + marker_wrap=Style.BRIGHT + '{}' + Style.RESET_ALL, + )), + ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', + ] + else: + widgets = [ + ' [Epoch ', progressbar.Counter(), ' of ', str(self.total_epochs), + ' (', progressbar.Percentage(), ')] ', + progressbar.Bar(marker=progressbar.AnimatedMarker( + fill='\N{FULL BLOCK}', + )), + ' [', progressbar.ETA(), ', ', progressbar.Timer(), ']', + ] + with progressbar.ProgressBar( + max_value=self.total_epochs, redirect_stdout=False, redirect_stderr=False, + widgets=widgets + ) as pbar: + EVALS = ceil(self.total_epochs / jobs) + for i in range(EVALS): + # Correct the number of epochs to be processed for the last + # iteration (should not exceed self.total_epochs in total) + n_rest = (i + 1) * jobs - self.total_epochs + current_jobs = jobs - n_rest if n_rest > 0 else jobs - self.print_results(val) + asked = self.opt.ask(n_points=current_jobs) + f_val = self.run_optimizer_parallel(parallel, asked, i) + self.opt.tell(asked, [v['loss'] for v in f_val]) + + # Calculate progressbar outputs + for j, val in enumerate(f_val): + # Use human-friendly indexes here (starting from 1) + current = i * jobs + j + 1 + val['current_epoch'] = current + val['is_initial_point'] = current <= INITIAL_POINTS + + logger.debug(f"Optimizer epoch evaluated: {val}") + + is_best = self.is_best_loss(val, self.current_best_loss) + # This value is assigned here and not in the optimization method + # to keep proper order in the list of results. That's because + # evaluations can take different time. Here they are aligned in the + # order they will be shown to the user. + val['is_best'] = is_best + self.print_results(val) + + if is_best: + self.current_best_loss = val['loss'] + self.epochs.append(val) + + # Save results after each best epoch and every 100 epochs + if is_best or current % 100 == 0: + self._save_results() + + pbar.update(current) - if is_best: - self.current_best_loss = val['loss'] - self.trials.append(val) - # Save results after each best epoch and every 100 epochs - if is_best or current % 100 == 0: - self.save_trials() except KeyboardInterrupt: print('User interrupted..') - self.save_trials(final=True) + self._save_results() + logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} " + f"saved to '{self.results_file}'.") - if self.trials: - sorted_trials = sorted(self.trials, key=itemgetter('loss')) - results = sorted_trials[0] - self.print_epoch_details(results, self.total_epochs, self.print_json) + if self.epochs: + sorted_epochs = sorted(self.epochs, key=itemgetter('loss')) + best_epoch = sorted_epochs[0] + self.print_epoch_details(best_epoch, self.total_epochs, self.print_json) else: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 856f3eee7..b3cedef2c 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -4,17 +4,15 @@ This module defines the interface to apply for hyperopt """ import logging import math - from abc import ABC -from typing import Dict, Any, Callable, List +from typing import Any, Callable, Dict, List from skopt.space import Categorical, Dimension, Integer, Real -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import round_dict - logger = logging.getLogger(__name__) @@ -209,7 +207,7 @@ class IHyperOpt(ABC): # so this intermediate parameter is used as the value of the difference between # them. The value of the 'trailing_stop_positive_offset' is constructed in the # generate_trailing_params() method. - # # This is similar to the hyperspace dimensions used for constructing the ROI tables. + # This is similar to the hyperspace dimensions used for constructing the ROI tables. Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'), Categorical([True, False], name='trailing_only_offset_is_reached'), diff --git a/freqtrade/optimize/hyperopt_loss_sharpe.py b/freqtrade/optimize/hyperopt_loss_sharpe.py index 5631a75de..29377bdd5 100644 --- a/freqtrade/optimize/hyperopt_loss_sharpe.py +++ b/freqtrade/optimize/hyperopt_loss_sharpe.py @@ -28,18 +28,19 @@ class SharpeHyperOptLoss(IHyperOptLoss): Uses Sharpe Ratio calculation. """ - total_profit = results.profit_percent + total_profit = results["profit_percent"] days_period = (max_date - min_date).days # adding slippage of 0.1% per trade total_profit = total_profit - 0.0005 - expected_yearly_return = total_profit.sum() / days_period + expected_returns_mean = total_profit.sum() / days_period + up_stdev = np.std(total_profit) - if (np.std(total_profit) != 0.): - sharp_ratio = expected_yearly_return / np.std(total_profit) * np.sqrt(365) + if up_stdev != 0: + sharp_ratio = expected_returns_mean / up_stdev * np.sqrt(365) else: # Define high (negative) sharpe ratio to be clear that this is NOT optimal. sharp_ratio = -20. - # print(expected_yearly_return, np.std(total_profit), sharp_ratio) + # print(expected_returns_mean, up_stdev, sharp_ratio) return -sharp_ratio diff --git a/freqtrade/optimize/hyperopt_loss_sharpe_daily.py b/freqtrade/optimize/hyperopt_loss_sharpe_daily.py new file mode 100644 index 000000000..e4cd1d749 --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sharpe_daily.py @@ -0,0 +1,62 @@ +""" +SharpeHyperOptLossDaily + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +import math +from datetime import datetime + +from pandas import DataFrame, date_range + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SharpeHyperOptLossDaily(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sharpe Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sharpe Ratio calculation. + """ + resample_freq = '1D' + slippage_per_trade_ratio = 0.0005 + days_in_year = 365 + annual_risk_free_rate = 0.0 + risk_free_rate = annual_risk_free_rate / days_in_year + + # apply slippage per trade to profit_percent + results.loc[:, 'profit_percent_after_slippage'] = \ + results['profit_percent'] - slippage_per_trade_ratio + + # create the index within the min_date and end max_date + t_index = date_range(start=min_date, end=max_date, freq=resample_freq, + normalize=True) + + sum_daily = ( + results.resample(resample_freq, on='close_time').agg( + {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0) + ) + + total_profit = sum_daily["profit_percent_after_slippage"] - risk_free_rate + expected_returns_mean = total_profit.mean() + up_stdev = total_profit.std() + + if up_stdev != 0: + sharp_ratio = expected_returns_mean / up_stdev * math.sqrt(days_in_year) + else: + # Define high (negative) sharpe ratio to be clear that this is NOT optimal. + sharp_ratio = -20. + + # print(t_index, sum_daily, total_profit) + # print(risk_free_rate, expected_returns_mean, up_stdev, sharp_ratio) + return -sharp_ratio diff --git a/freqtrade/optimize/hyperopt_loss_sortino.py b/freqtrade/optimize/hyperopt_loss_sortino.py new file mode 100644 index 000000000..d470a9977 --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sortino.py @@ -0,0 +1,49 @@ +""" +SortinoHyperOptLoss + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +from datetime import datetime + +from pandas import DataFrame +import numpy as np + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SortinoHyperOptLoss(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sortino Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sortino Ratio calculation. + """ + total_profit = results["profit_percent"] + days_period = (max_date - min_date).days + + # adding slippage of 0.1% per trade + total_profit = total_profit - 0.0005 + expected_returns_mean = total_profit.sum() / days_period + + results['downside_returns'] = 0 + results.loc[total_profit < 0, 'downside_returns'] = results['profit_percent'] + down_stdev = np.std(results['downside_returns']) + + if down_stdev != 0: + sortino_ratio = expected_returns_mean / down_stdev * np.sqrt(365) + else: + # Define high (negative) sortino ratio to be clear that this is NOT optimal. + sortino_ratio = -20. + + # print(expected_returns_mean, down_stdev, sortino_ratio) + return -sortino_ratio diff --git a/freqtrade/optimize/hyperopt_loss_sortino_daily.py b/freqtrade/optimize/hyperopt_loss_sortino_daily.py new file mode 100644 index 000000000..cd6a8bcc2 --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_sortino_daily.py @@ -0,0 +1,70 @@ +""" +SortinoHyperOptLossDaily + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +import math +from datetime import datetime + +from pandas import DataFrame, date_range + +from freqtrade.optimize.hyperopt import IHyperOptLoss + + +class SortinoHyperOptLossDaily(IHyperOptLoss): + """ + Defines the loss function for hyperopt. + + This implementation uses the Sortino Ratio calculation. + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + """ + Objective function, returns smaller number for more optimal results. + + Uses Sortino Ratio calculation. + + Sortino Ratio calculated as described in + http://www.redrockcapital.com/Sortino__A__Sharper__Ratio_Red_Rock_Capital.pdf + """ + resample_freq = '1D' + slippage_per_trade_ratio = 0.0005 + days_in_year = 365 + minimum_acceptable_return = 0.0 + + # apply slippage per trade to profit_percent + results.loc[:, 'profit_percent_after_slippage'] = \ + results['profit_percent'] - slippage_per_trade_ratio + + # create the index within the min_date and end max_date + t_index = date_range(start=min_date, end=max_date, freq=resample_freq, + normalize=True) + + sum_daily = ( + results.resample(resample_freq, on='close_time').agg( + {"profit_percent_after_slippage": sum}).reindex(t_index).fillna(0) + ) + + total_profit = sum_daily["profit_percent_after_slippage"] - minimum_acceptable_return + expected_returns_mean = total_profit.mean() + + sum_daily['downside_returns'] = 0 + sum_daily.loc[total_profit < 0, 'downside_returns'] = total_profit + total_downside = sum_daily['downside_returns'] + # Here total_downside contains min(0, P - MAR) values, + # where P = sum_daily["profit_percent_after_slippage"] + down_stdev = math.sqrt((total_downside**2).sum() / len(total_downside)) + + if down_stdev != 0: + sortino_ratio = expected_returns_mean / down_stdev * math.sqrt(days_in_year) + else: + # Define high (negative) sortino ratio to be clear that this is NOT optimal. + sortino_ratio = -20. + + # print(t_index, sum_daily, total_profit) + # print(minimum_acceptable_return, expected_returns_mean, down_stdev, sortino_ratio) + return -sortino_ratio diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py new file mode 100644 index 000000000..646afb5df --- /dev/null +++ b/freqtrade/optimize/optimize_reports.py @@ -0,0 +1,244 @@ +import logging +from datetime import timedelta +from pathlib import Path +from typing import Dict + +from pandas import DataFrame +from tabulate import tabulate + +from freqtrade.misc import file_dump_json + +logger = logging.getLogger(__name__) + + +def store_backtest_result(recordfilename: Path, all_results: Dict[str, DataFrame]) -> None: + """ + Stores backtest results to file (one file per strategy) + :param recordfilename: Destination filename + :param all_results: Dict of Dataframes, one results dataframe per strategy + """ + for strategy, results in all_results.items(): + records = [(t.pair, t.profit_percent, t.open_time.timestamp(), + t.close_time.timestamp(), t.open_index - 1, t.trade_duration, + t.open_rate, t.close_rate, t.open_at_end, t.sell_reason.value) + for index, t in results.iterrows()] + + if records: + filename = recordfilename + if len(all_results) > 1: + # Inject strategy to filename + filename = Path.joinpath( + recordfilename.parent, + f'{recordfilename.stem}-{strategy}').with_suffix(recordfilename.suffix) + logger.info(f'Dumping backtest results to {filename}') + file_dump_json(filename, records) + + +def generate_text_table(data: Dict[str, Dict], stake_currency: str, max_open_trades: int, + results: DataFrame, skip_nan: bool = False) -> str: + """ + Generates and returns a text table for the given backtest data and the results dataframe + :param data: Dict of containing data that was used during backtesting. + :param stake_currency: stake-currency - used to correctly name headers + :param max_open_trades: Maximum allowed open trades + :param results: Dataframe containing the backtest results + :param skip_nan: Print "left open" open trades + :return: pretty printed table with tabulate as string + """ + + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f') + tabular_data = [] + headers = [ + 'Pair', + 'Buys', + 'Avg Profit %', + 'Cum Profit %', + f'Tot Profit {stake_currency}', + 'Tot Profit %', + 'Avg Duration', + 'Wins', + 'Draws', + 'Losses' + ] + for pair in data: + result = results[results.pair == pair] + if skip_nan and result.profit_abs.isnull().all(): + continue + + tabular_data.append([ + pair, + len(result.index), + result.profit_percent.mean() * 100.0, + result.profit_percent.sum() * 100.0, + result.profit_abs.sum(), + result.profit_percent.sum() * 100.0 / max_open_trades, + str(timedelta( + minutes=round(result.trade_duration.mean()))) if not result.empty else '0:00', + len(result[result.profit_abs > 0]), + len(result[result.profit_abs == 0]), + len(result[result.profit_abs < 0]) + ]) + + # Append Total + tabular_data.append([ + 'TOTAL', + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_percent.sum() * 100.0, + results.profit_abs.sum(), + results.profit_percent.sum() * 100.0 / max_open_trades, + str(timedelta( + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', + len(results[results.profit_abs > 0]), + len(results[results.profit_abs == 0]), + len(results[results.profit_abs < 0]) + ]) + # Ignore type as floatfmt does allow tuples but mypy does not know that + return tabulate(tabular_data, headers=headers, + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore + + +def generate_text_table_sell_reason(stake_currency: str, max_open_trades: int, + results: DataFrame) -> str: + """ + Generate small table outlining Backtest results + :param stake_currency: Stakecurrency used + :param max_open_trades: Max_open_trades parameter + :param results: Dataframe containing the backtest results + :return: pretty printed table with tabulate as string + """ + tabular_data = [] + headers = [ + "Sell Reason", + "Sells", + "Wins", + "Draws", + "Losses", + "Avg Profit %", + "Cum Profit %", + f"Tot Profit {stake_currency}", + "Tot Profit %", + ] + for reason, count in results['sell_reason'].value_counts().iteritems(): + result = results.loc[results['sell_reason'] == reason] + wins = len(result[result['profit_abs'] > 0]) + draws = len(result[result['profit_abs'] == 0]) + loss = len(result[result['profit_abs'] < 0]) + profit_mean = round(result['profit_percent'].mean() * 100.0, 2) + profit_sum = round(result["profit_percent"].sum() * 100.0, 2) + profit_tot = result['profit_abs'].sum() + profit_percent_tot = round(result['profit_percent'].sum() * 100.0 / max_open_trades, 2) + tabular_data.append( + [ + reason.value, + count, + wins, + draws, + loss, + profit_mean, + profit_sum, + profit_tot, + profit_percent_tot, + ] + ) + return tabulate(tabular_data, headers=headers, tablefmt="orgtbl", stralign="right") + + +def generate_text_table_strategy(stake_currency: str, max_open_trades: str, + all_results: Dict) -> str: + """ + Generate summary table per strategy + :param stake_currency: stake-currency - used to correctly name headers + :param max_open_trades: Maximum allowed open trades used for backtest + :param all_results: Dict of containing results for all strategies + :return: pretty printed table with tabulate as string + """ + + floatfmt = ('s', 'd', '.2f', '.2f', '.8f', '.2f', 'd', '.1f', '.1f') + tabular_data = [] + headers = ['Strategy', 'Buys', 'Avg Profit %', 'Cum Profit %', + f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration', + 'Wins', 'Draws', 'Losses'] + for strategy, results in all_results.items(): + tabular_data.append([ + strategy, + len(results.index), + results.profit_percent.mean() * 100.0, + results.profit_percent.sum() * 100.0, + results.profit_abs.sum(), + results.profit_percent.sum() * 100.0 / max_open_trades, + str(timedelta( + minutes=round(results.trade_duration.mean()))) if not results.empty else '0:00', + len(results[results.profit_abs > 0]), + len(results[results.profit_abs == 0]), + len(results[results.profit_abs < 0]) + ]) + # Ignore type as floatfmt does allow tuples but mypy does not know that + return tabulate(tabular_data, headers=headers, + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore + + +def generate_edge_table(results: dict) -> str: + + floatfmt = ('s', '.10g', '.2f', '.2f', '.2f', '.2f', 'd', '.d') + tabular_data = [] + headers = ['Pair', 'Stoploss', 'Win Rate', 'Risk Reward Ratio', + 'Required Risk Reward', 'Expectancy', 'Total Number of Trades', + 'Average Duration (min)'] + + for result in results.items(): + if result[1].nb_trades > 0: + tabular_data.append([ + result[0], + result[1].stoploss, + result[1].winrate, + result[1].risk_reward_ratio, + result[1].required_risk_reward, + result[1].expectancy, + result[1].nb_trades, + round(result[1].avg_trade_duration) + ]) + + # Ignore type as floatfmt does allow tuples but mypy does not know that + return tabulate(tabular_data, headers=headers, + floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore + + +def show_backtest_results(config: Dict, btdata: Dict[str, DataFrame], + all_results: Dict[str, DataFrame]): + for strategy, results in all_results.items(): + + print(f"Result for strategy {strategy}") + table = generate_text_table(btdata, stake_currency=config['stake_currency'], + max_open_trades=config['max_open_trades'], + results=results) + if isinstance(table, str): + print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + + table = generate_text_table_sell_reason(stake_currency=config['stake_currency'], + max_open_trades=config['max_open_trades'], + results=results) + if isinstance(table, str): + print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '=')) + print(table) + + table = generate_text_table(btdata, + stake_currency=config['stake_currency'], + max_open_trades=config['max_open_trades'], + results=results.loc[results.open_at_end], skip_nan=True) + if isinstance(table, str): + print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '=')) + print(table) + if isinstance(table, str): + print('=' * len(table.splitlines()[0])) + print() + if len(all_results) > 1: + # Print Strategy summary table + table = generate_text_table_strategy(config['stake_currency'], + config['max_open_trades'], + all_results=all_results) + print(' STRATEGY SUMMARY '.center(len(table.splitlines()[0]), '=')) + print(table) + print('=' * len(table.splitlines()[0])) + print('\nFor more details, please look at the detail tables above') diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/pairlist/IPairList.py index d722e70f5..e2eb364bc 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/pairlist/IPairList.py @@ -1,22 +1,23 @@ """ -Static List provider - -Provides lists as configured in config.json - - """ +PairList base class +""" import logging from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy -from typing import Dict, List +from typing import Any, Dict, List + +from cachetools import TTLCache, cached from freqtrade.exchange import market_is_active + logger = logging.getLogger(__name__) class IPairList(ABC): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: """ :param exchange: Exchange instance @@ -30,6 +31,9 @@ class IPairList(ABC): self._config = config self._pairlistconfig = pairlistconfig self._pairlist_pos = pairlist_pos + self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) + self._last_refresh = 0 + self._log_cache = TTLCache(maxsize=1024, ttl=self.refresh_period) @property def name(self) -> str: @@ -39,6 +43,24 @@ class IPairList(ABC): """ return self.__class__.__name__ + def log_on_refresh(self, logmethod, message: str) -> None: + """ + Logs message - not more often than "refresh_period" to avoid log spamming + Logs the log-message as debug as well to simplify debugging. + :param logmethod: Function that'll be called. Most likely `logger.info`. + :param message: String containing the message to be sent to the function. + :return: None. + """ + + @cached(cache=self._log_cache) + def _log_on_refresh(message: str): + logmethod(message) + + # Log as debug first + logger.debug(message) + # Call hidden function. + _log_on_refresh(message) + @abstractproperty def needstickers(self) -> bool: """ @@ -66,21 +88,37 @@ class IPairList(ABC): """ @staticmethod - def verify_blacklist(pairlist: List[str], blacklist: List[str]) -> List[str]: + def verify_blacklist(pairlist: List[str], blacklist: List[str], + aswarning: bool) -> List[str]: """ Verify and remove items from pairlist - returning a filtered pairlist. + Logs a warning or info depending on `aswarning`. + Pairlists explicitly using this method shall use `aswarning=False`! + :param pairlist: Pairlist to validate + :param blacklist: Blacklist to validate pairlist against + :param aswarning: Log message as Warning or info + :return: pairlist - blacklisted pairs """ for pair in deepcopy(pairlist): if pair in blacklist: - logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...") + if aswarning: + logger.warning(f"Pair {pair} in your blacklist. Removing it from whitelist...") + else: + logger.info(f"Pair {pair} in your blacklist. Removing it from whitelist...") pairlist.remove(pair) return pairlist - def _verify_blacklist(self, pairlist: List[str]) -> List[str]: + def _verify_blacklist(self, pairlist: List[str], aswarning: bool = True) -> List[str]: """ Proxy method to verify_blacklist for easy access for child classes. + Logs a warning or info depending on `aswarning`. + Pairlists explicitly using this method shall use aswarning=False! + :param pairlist: Pairlist to validate + :param aswarning: Log message as Warning or info. + :return: pairlist - blacklisted pairs """ - return IPairList.verify_blacklist(pairlist, self._pairlistmanager.blacklist) + return IPairList.verify_blacklist(pairlist, self._pairlistmanager.blacklist, + aswarning=aswarning) def _whitelist_for_active_markets(self, pairlist: List[str]) -> List[str]: """ @@ -98,7 +136,8 @@ class IPairList(ABC): logger.warning(f"Pair {pair} is not compatible with exchange " f"{self._exchange.name}. Removing it from whitelist..") continue - if not pair.endswith(self._config['stake_currency']): + + if self._exchange.get_pair_quote_currency(pair) != self._config['stake_currency']: logger.warning(f"Pair {pair} is not compatible with your stake currency " f"{self._config['stake_currency']}. Removing it from whitelist..") continue @@ -111,6 +150,5 @@ class IPairList(ABC): if pair not in sanitized_whitelist: sanitized_whitelist.append(pair) - sanitized_whitelist = self._verify_blacklist(sanitized_whitelist) # We need to remove pairs that are unknown return sanitized_whitelist diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/pairlist/PrecisionFilter.py index aedcc5a88..6bd9c594e 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/pairlist/PrecisionFilter.py @@ -1,14 +1,26 @@ +""" +Precision pair list filter +""" import logging from copy import deepcopy -from typing import Dict, List +from typing import Any, Dict, List from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) class PrecisionFilter(IPairList): + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + # Precalculate sanitized stoploss value to avoid recalculation for every pair + self._stoploss = 1 - abs(self._config['stoploss']) + @property def needstickers(self) -> bool: """ @@ -31,33 +43,32 @@ class PrecisionFilter(IPairList): :param ticker: ticker dict as returned from ccxt.load_markets() :param stoploss: stoploss value as set in the configuration (already cleaned to be 1 - stoploss) - :return: True if the pair can stay, false if it should be removed + :return: True if the pair can stay, False if it should be removed """ stop_price = ticker['ask'] * stoploss + # Adjust stop-prices to precision - sp = self._exchange.symbol_price_prec(ticker["symbol"], stop_price) - stop_gap_price = self._exchange.symbol_price_prec(ticker["symbol"], stop_price * 0.99) + sp = self._exchange.price_to_precision(ticker["symbol"], stop_price) + + stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99) logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") + if sp <= stop_gap_price: - logger.info(f"Removed {ticker['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {stop_gap_price}") + self.log_on_refresh(logger.info, + f"Removed {ticker['symbol']} from whitelist, " + f"because stop price {sp} would be <= stop limit {stop_gap_price}") return False + return True def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Filters and sorts pairlists and assigns and returns them again. """ - stoploss = None - if self._config.get('stoploss') is not None: - # Precalculate sanitized stoploss value to avoid recalculation for every pair - stoploss = 1 - abs(self._config.get('stoploss')) # Copy list since we're modifying this list for p in deepcopy(pairlist): - ticker = tickers.get(p) # Filter out assets which would not allow setting a stoploss - if not ticker or (stoploss and not self._validate_precision_filter(ticker, stoploss)): + if not self._validate_precision_filter(tickers[p], self._stoploss): pairlist.remove(p) - continue return pairlist diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/pairlist/PriceFilter.py index b3546ebd9..167717656 100644 --- a/freqtrade/pairlist/PriceFilter.py +++ b/freqtrade/pairlist/PriceFilter.py @@ -1,15 +1,20 @@ +""" +Price pair list filter +""" import logging from copy import deepcopy -from typing import Dict, List +from typing import Any, Dict, List from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) class PriceFilter(IPairList): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) @@ -34,21 +39,22 @@ class PriceFilter(IPairList): """ Check if if one price-step (pip) is > than a certain barrier. :param ticker: ticker dict as returned from ccxt.load_markets() - :param precision: Precision :return: True if the pair can stay, false if it should be removed """ - precision = self._exchange.markets[ticker['symbol']]['precision']['price'] - - compare = ticker['last'] + 1 / pow(10, precision) - changeperc = (compare - ticker['last']) / ticker['last'] + if ticker['last'] is None: + self.log_on_refresh(logger.info, + f"Removed {ticker['symbol']} from whitelist, because " + "ticker['last'] is empty (Usually no trade in the last 24h).") + return False + compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last']) + changeperc = compare / ticker['last'] if changeperc > self._low_price_ratio: - logger.info(f"Removed {ticker['symbol']} from whitelist, " - f"because 1 unit is {changeperc * 100:.3f}%") + self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%") return False return True def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: - """ Filters and sorts pairlist and returns the whitelist again. Called on each bot iteration - please use internal caching if necessary @@ -56,14 +62,11 @@ class PriceFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new whitelist """ - # Copy list since we're modifying this list - for p in deepcopy(pairlist): - ticker = tickers.get(p) - if not ticker: - pairlist.remove(p) - - # Filter out assets which would not allow setting a stoploss - if self._low_price_ratio and not self._validate_ticker_lowprice(ticker): - pairlist.remove(p) + if self._low_price_ratio: + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + # Filter out assets which would not allow setting a stoploss + if not self._validate_ticker_lowprice(tickers[p]): + pairlist.remove(p) return pairlist diff --git a/freqtrade/pairlist/SpreadFilter.py b/freqtrade/pairlist/SpreadFilter.py new file mode 100644 index 000000000..88e143a50 --- /dev/null +++ b/freqtrade/pairlist/SpreadFilter.py @@ -0,0 +1,70 @@ +""" +Spread pair list filter +""" +import logging +from copy import deepcopy +from typing import Dict, List + +from freqtrade.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class SpreadFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._max_spread_ratio = pairlistconfig.get('max_spread_ratio', 0.005) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return True + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return (f"{self.name} - Filtering pairs with ask/bid diff above " + f"{self._max_spread_ratio * 100}%.") + + def _validate_spread(self, ticker: dict) -> bool: + """ + Validate spread for the ticker + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, False if it should be removed + """ + if 'bid' in ticker and 'ask' in ticker: + spread = 1 - ticker['bid'] / ticker['ask'] + if spread > self._max_spread_ratio: + self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " + f"because spread {spread * 100:.3f}% >" + f"{self._max_spread_ratio * 100}%") + return False + else: + return True + return False + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + ticker = tickers[p] + # Filter out assets + if not self._validate_spread(ticker): + pairlist.remove(p) + + return pairlist diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/pairlist/StaticPairList.py index 0050fbd5c..07e559168 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/pairlist/StaticPairList.py @@ -1,14 +1,14 @@ """ -Static List provider +Static Pair List provider -Provides lists as configured in config.json - - """ +Provides pair white list as it configured in config +""" import logging from typing import Dict, List from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/pairlist/VolumePairList.py index 2df9ba691..981e9915e 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/pairlist/VolumePairList.py @@ -1,33 +1,37 @@ """ Volume PairList provider -Provides lists as configured in config.json - - """ +Provides dynamic pair list based on trade volumes +""" import logging from datetime import datetime -from typing import Dict, List +from typing import Any, Dict, List -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList + logger = logging.getLogger(__name__) + SORT_VALUES = ['askVolume', 'bidVolume', 'quoteVolume'] class VolumePairList(IPairList): - def __init__(self, exchange, pairlistmanager, config, pairlistconfig: dict, + def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: dict, pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) if 'number_assets' not in self._pairlistconfig: raise OperationalException( - f'`number_assets` not specified. Please check your configuration ' + '`number_assets` not specified. Please check your configuration ' 'for "pairlist.config.number_assets"') + + self._stake_currency = config['stake_currency'] self._number_pairs = self._pairlistconfig['number_assets'] self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume') + self._min_value = self._pairlistconfig.get('min_value', 0) self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) if not self._exchange.exchange_has('fetchTickers'): @@ -35,10 +39,15 @@ class VolumePairList(IPairList): 'Exchange does not support dynamic whitelist.' 'Please edit your config and restart the bot' ) + if not self._validate_keys(self._sort_key): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') - self._last_refresh = 0 + + if self._sort_key != 'quoteVolume': + logger.warning( + "DEPRECATED: using any key other than quoteVolume for VolumePairList is deprecated." + ) @property def needstickers(self) -> bool: @@ -67,42 +76,47 @@ class VolumePairList(IPairList): :return: new whitelist """ # Generate dynamic whitelist - if self._last_refresh + self.refresh_period < datetime.now().timestamp(): - self._last_refresh = int(datetime.now().timestamp()) - return self._gen_pair_whitelist(pairlist, - tickers, - self._config['stake_currency'], - self._sort_key, - ) - else: - return pairlist + # Must always run if this pairlist is not the first in the list. + if (self._pairlist_pos != 0 or + (self._last_refresh + self.refresh_period < datetime.now().timestamp())): - def _gen_pair_whitelist(self, pairlist, tickers, base_currency: str, key: str) -> List[str]: + self._last_refresh = int(datetime.now().timestamp()) + pairs = self._gen_pair_whitelist(pairlist, tickers) + else: + pairs = pairlist + + self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") + + return pairs + + def _gen_pair_whitelist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Updates the whitelist with with a dynamically generated list - :param base_currency: base currency as str - :param key: sort key (defaults to 'quoteVolume') + :param pairlist: pairlist to filter or sort :param tickers: Tickers (from exchange.get_tickers()). :return: List of pairs """ - if self._pairlist_pos == 0: # If VolumePairList is the first in the list, use fresh pairlist - # check length so that we make sure that '/' is actually in the string - filtered_tickers = [v for k, v in tickers.items() - if (len(k.split('/')) == 2 and k.split('/')[1] == base_currency - and v[key] is not None)] + # Check if pair quote currency equals to the stake currency. + filtered_tickers = [ + v for k, v in tickers.items() + if (self._exchange.get_pair_quote_currency(k) == self._stake_currency + and v[self._sort_key] is not None)] else: - # If other pairlist is in front, use the incomming pairlist. + # If other pairlist is in front, use the incoming pairlist. filtered_tickers = [v for k, v in tickers.items() if k in pairlist] - sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[key]) + if self._min_value > 0: + filtered_tickers = [ + v for v in filtered_tickers if v[self._sort_key] > self._min_value] + + sorted_tickers = sorted(filtered_tickers, reverse=True, key=lambda t: t[self._sort_key]) # Validate whitelist to only have active market pairs pairs = self._whitelist_for_active_markets([s['symbol'] for s in sorted_tickers]) - pairs = self._verify_blacklist(pairs) - # Limit to X number of pairs + pairs = self._verify_blacklist(pairs, aswarning=False) + # Limit pairlist to the requested number of pairs pairs = pairs[:self._number_pairs] - logger.info(f"Searching {self._number_pairs} pairs: {pairs}") return pairs diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index fa5382c37..b01f1342b 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -1,17 +1,17 @@ """ -Static List provider - -Provides lists as configured in config.json - - """ -from cachetools import TTLCache, cached +PairList manager class +""" import logging +from copy import deepcopy from typing import Dict, List -from freqtrade import OperationalException +from cachetools import TTLCache, cached + +from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import PairListResolver + logger = logging.getLogger(__name__) @@ -28,13 +28,13 @@ class PairListManager(): if 'method' not in pl: logger.warning(f"No method in {pl}") continue - pairl = PairListResolver(pl.get('method'), - exchange=exchange, - pairlistmanager=self, - config=config, - pairlistconfig=pl, - pairlist_pos=len(self._pairlists) - ).pairlist + pairl = PairListResolver.load_pairlist(pl.get('method'), + exchange=exchange, + pairlistmanager=self, + config=config, + pairlistconfig=pl, + pairlist_pos=len(self._pairlists) + ) self._tickers_needed = pairl.needstickers or self._tickers_needed self._pairlists.append(pairl) @@ -77,19 +77,33 @@ class PairListManager(): """ Run pairlist through all configured pairlists. """ - - pairlist = self._whitelist.copy() - - # tickers should be cached to avoid calling the exchange on each call. + # Tickers should be cached to avoid calling the exchange on each call. tickers: Dict = {} if self._tickers_needed: tickers = self._get_cached_tickers() + # Adjust whitelist if filters are using tickers + pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers) + # Process all pairlists in chain for pl in self._pairlists: pairlist = pl.filter_pairlist(pairlist, tickers) - # Validation against blacklist happens after the pairlists to ensure blacklist is respected. - pairlist = IPairList.verify_blacklist(pairlist, self.blacklist) + # Validation against blacklist happens after the pairlists to ensure + # blacklist is respected. + pairlist = IPairList.verify_blacklist(pairlist, self.blacklist, True) self._whitelist = pairlist + + def _prepare_whitelist(self, pairlist: List[str], tickers) -> List[str]: + """ + Prepare sanitized pairlist for Pairlist Filters that use tickers data - remove + pairs that do not have ticker available + """ + if self._tickers_needed: + # Copy list since we're modifying this list + for p in deepcopy(pairlist): + if p not in tickers: + pairlist.remove(p) + + return pairlist diff --git a/freqtrade/persistence.py b/freqtrade/persistence.py index 993b68bc7..3f7f4e0e9 100644 --- a/freqtrade/persistence.py +++ b/freqtrade/persistence.py @@ -16,7 +16,7 @@ from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) @@ -64,11 +64,11 @@ def init(db_url: str, clean_open_orders: bool = False) -> None: clean_dry_run_db() -def has_column(columns, searchname: str) -> bool: +def has_column(columns: List, searchname: str) -> bool: return len(list(filter(lambda x: x["name"] == searchname, columns))) == 1 -def get_column_def(columns, column: str, default: str) -> str: +def get_column_def(columns: List, column: str, default: str) -> str: return default if not has_column(columns, column) else column @@ -86,11 +86,15 @@ def check_migrate(engine) -> None: logger.debug(f'trying {table_back_name}') # Check for latest column - if not has_column(cols, 'open_trade_price'): + if not has_column(cols, 'sell_order_status'): logger.info(f'Running database migration - backup available as {table_back_name}') fee_open = get_column_def(cols, 'fee_open', 'fee') + fee_open_cost = get_column_def(cols, 'fee_open_cost', 'null') + fee_open_currency = get_column_def(cols, 'fee_open_currency', 'null') fee_close = get_column_def(cols, 'fee_close', 'fee') + fee_close_cost = get_column_def(cols, 'fee_close_cost', 'null') + fee_close_currency = get_column_def(cols, 'fee_close_currency', 'null') open_rate_requested = get_column_def(cols, 'open_rate_requested', 'null') close_rate_requested = get_column_def(cols, 'close_rate_requested', 'null') stop_loss = get_column_def(cols, 'stop_loss', '0.0') @@ -106,6 +110,10 @@ def check_migrate(engine) -> None: ticker_interval = get_column_def(cols, 'ticker_interval', 'null') open_trade_price = get_column_def(cols, 'open_trade_price', f'amount * open_rate * (1 + {fee_open})') + close_profit_abs = get_column_def( + cols, 'close_profit_abs', + f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") + sell_order_status = get_column_def(cols, 'sell_order_status', 'null') # Schema migration necessary engine.execute(f"alter table trades rename to {table_back_name}") @@ -117,13 +125,15 @@ def check_migrate(engine) -> None: # Copy data back - following the correct schema engine.execute(f"""insert into trades - (id, exchange, pair, is_open, fee_open, fee_close, open_rate, + (id, exchange, pair, is_open, + fee_open, fee_open_cost, fee_open_currency, + fee_close, fee_close_cost, fee_open_currency, open_rate, open_rate_requested, close_rate, close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, stoploss_order_id, stoploss_last_update, - max_rate, min_rate, sell_reason, strategy, - ticker_interval, open_trade_price + max_rate, min_rate, sell_reason, sell_order_status, strategy, + ticker_interval, open_trade_price, close_profit_abs ) select id, lower(exchange), case @@ -133,7 +143,9 @@ def check_migrate(engine) -> None: else pair end pair, - is_open, {fee_open} fee_open, {fee_close} fee_close, + is_open, {fee_open} fee_open, {fee_open_cost} fee_open_cost, + {fee_open_currency} fee_open_currency, {fee_close} fee_close, + {fee_close_cost} fee_close_cost, {fee_close_currency} fee_close_currency, open_rate, {open_rate_requested} open_rate_requested, close_rate, {close_rate_requested} close_rate_requested, close_profit, stake_amount, amount, open_date, close_date, open_order_id, @@ -142,8 +154,9 @@ def check_migrate(engine) -> None: {initial_stop_loss_pct} initial_stop_loss_pct, {stoploss_order_id} stoploss_order_id, {stoploss_last_update} stoploss_last_update, {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, + {sell_order_status} sell_order_status, {strategy} strategy, {ticker_interval} ticker_interval, - {open_trade_price} open_trade_price + {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs from {table_back_name} """) @@ -182,14 +195,19 @@ class Trade(_DECL_BASE): pair = Column(String, nullable=False, index=True) is_open = Column(Boolean, nullable=False, default=True, index=True) fee_open = Column(Float, nullable=False, default=0.0) + fee_open_cost = Column(Float, nullable=True) + fee_open_currency = Column(String, nullable=True) fee_close = Column(Float, nullable=False, default=0.0) + fee_close_cost = Column(Float, nullable=True) + fee_close_currency = Column(String, nullable=True) open_rate = Column(Float) open_rate_requested = Column(Float) - # open_trade_price - calcuated via _calc_open_trade_price + # open_trade_price - calculated via _calc_open_trade_price open_trade_price = Column(Float) close_rate = Column(Float) close_rate_requested = Column(Float) close_profit = Column(Float) + close_profit_abs = Column(Float) stake_amount = Column(Float, nullable=False) amount = Column(Float) open_date = Column(DateTime, nullable=False, default=datetime.utcnow) @@ -212,6 +230,7 @@ class Trade(_DECL_BASE): # Lowest price reached min_rate = Column(Float, nullable=True) sell_reason = Column(String, nullable=True) + sell_order_status = Column(String, nullable=True) strategy = Column(String, nullable=True) ticker_interval = Column(Integer, nullable=True) @@ -229,6 +248,13 @@ class Trade(_DECL_BASE): return { 'trade_id': self.id, 'pair': self.pair, + 'is_open': self.is_open, + 'fee_open': self.fee_open, + 'fee_open_cost': self.fee_open_cost, + 'fee_open_currency': self.fee_open_currency, + 'fee_close': self.fee_close, + 'fee_close_cost': self.fee_close_cost, + 'fee_close_currency': self.fee_close_currency, 'open_date_hum': arrow.get(self.open_date).humanize(), 'open_date': self.open_date.strftime("%Y-%m-%d %H:%M:%S"), 'close_date_hum': (arrow.get(self.close_date).humanize() @@ -236,24 +262,36 @@ class Trade(_DECL_BASE): 'close_date': (self.close_date.strftime("%Y-%m-%d %H:%M:%S") if self.close_date else None), 'open_rate': self.open_rate, + 'open_rate_requested': self.open_rate_requested, + 'open_trade_price': self.open_trade_price, 'close_rate': self.close_rate, + 'close_rate_requested': self.close_rate_requested, 'amount': round(self.amount, 8), 'stake_amount': round(self.stake_amount, 8), + 'close_profit': self.close_profit, + 'sell_reason': self.sell_reason, + 'sell_order_status': self.sell_order_status, 'stop_loss': self.stop_loss, 'stop_loss_pct': (self.stop_loss_pct * 100) if self.stop_loss_pct else None, 'initial_stop_loss': self.initial_stop_loss, 'initial_stop_loss_pct': (self.initial_stop_loss_pct * 100 if self.initial_stop_loss_pct else None), + 'min_rate': self.min_rate, + 'max_rate': self.max_rate, + 'strategy': self.strategy, + 'ticker_interval': self.ticker_interval, + 'open_order_id': self.open_order_id, } - def adjust_min_max_rates(self, current_price: float): + def adjust_min_max_rates(self, current_price: float) -> None: """ Adjust the max_rate and min_rate. """ self.max_rate = max(current_price, self.max_rate or self.open_rate) self.min_rate = min(current_price, self.min_rate or self.open_rate) - def adjust_stop_loss(self, current_price: float, stoploss: float, initial: bool = False): + def adjust_stop_loss(self, current_price: float, stoploss: float, + initial: bool = False) -> None: """ This adjusts the stop loss to it's most recently observed setting :param current_price: Current rate the asset is traded @@ -310,17 +348,17 @@ class Trade(_DECL_BASE): if order_type in ('market', 'limit') and order['side'] == 'buy': # Update open rate and actual amount self.open_rate = Decimal(order['price']) - self.amount = Decimal(order['amount']) + self.amount = Decimal(order.get('filled', order['amount'])) self.recalc_open_trade_price() logger.info('%s_BUY has been fulfilled for %s.', order_type.upper(), self) self.open_order_id = None elif order_type in ('market', 'limit') and order['side'] == 'sell': self.close(order['price']) logger.info('%s_SELL has been fulfilled for %s.', order_type.upper(), self) - elif order_type == 'stop_loss_limit': + elif order_type in ('stop_loss_limit', 'stop-loss'): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss - logger.info('STOP_LOSS_LIMIT is hit for %s.', self) + logger.info('%s is hit for %s.', order_type.upper(), self) self.close(order['average']) else: raise ValueError(f'Unknown order type: {order_type}') @@ -333,14 +371,45 @@ class Trade(_DECL_BASE): """ self.close_rate = Decimal(rate) self.close_profit = self.calc_profit_ratio() + self.close_profit_abs = self.calc_profit() self.close_date = datetime.utcnow() self.is_open = False + self.sell_order_status = 'closed' self.open_order_id = None logger.info( 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', self ) + def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], + side: str) -> None: + """ + Update Fee parameters. Only acts once per side + """ + if side == 'buy' and self.fee_open_currency is None: + self.fee_open_cost = fee_cost + self.fee_open_currency = fee_currency + if fee_rate is not None: + self.fee_open = fee_rate + # Assume close-fee will fall into the same fee category and take an educated guess + self.fee_close = fee_rate + elif side == 'sell' and self.fee_close_currency is None: + self.fee_close_cost = fee_cost + self.fee_close_currency = fee_currency + if fee_rate is not None: + self.fee_close = fee_rate + + def fee_updated(self, side: str) -> bool: + """ + Verify if this side (buy / sell) has already been updated + """ + if side == 'buy': + return self.fee_open_currency is not None + elif side == 'sell': + return self.fee_close_currency is not None + else: + return False + def _calc_open_trade_price(self) -> float: """ Calculate the open_rate including open_fee. @@ -404,8 +473,8 @@ class Trade(_DECL_BASE): rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - profit_percent = (close_trade_price / self.open_trade_price) - 1 - return float(f"{profit_percent:.8f}") + profit_ratio = (close_trade_price / self.open_trade_price) - 1 + return float(f"{profit_ratio:.8f}") @staticmethod def get_trades(trade_filter=None) -> Query: diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 85089af9c..60f838db2 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -3,11 +3,16 @@ from pathlib import Path from typing import Any, Dict, List import pandas as pd + from freqtrade.configuration import TimeRange -from freqtrade.data import history -from freqtrade.data.btanalysis import (combine_tickers_with_mean, +from freqtrade.data.btanalysis import (calculate_max_drawdown, + combine_dataframes_with_mean, create_cum_profit, extract_trades_of_period, load_trades) +from freqtrade.data.converter import trim_dataframe +from freqtrade.exchange import timeframe_to_prev_date +from freqtrade.data.history import load_data +from freqtrade.misc import pair_to_filename from freqtrade.resolvers import StrategyResolver logger = logging.getLogger(__name__) @@ -25,7 +30,7 @@ except ImportError: def init_plotscript(config): """ Initialize objects needed for plotting - :return: Dict with tickers, trades and pairs + :return: Dict with candle (OHLCV) data, trades and pairs """ if "pairs" in config: @@ -36,39 +41,56 @@ def init_plotscript(config): # Set timerange to use timerange = TimeRange.parse_timerange(config.get("timerange")) - tickers = history.load_data( - datadir=Path(str(config.get("datadir"))), + data = load_data( + datadir=config.get("datadir"), pairs=pairs, timeframe=config.get('ticker_interval', '5m'), timerange=timerange, + data_format=config.get('dataformat_ohlcv', 'json'), ) - trades = load_trades(config['trade_source'], - db_url=config.get('db_url'), - exportfilename=config.get('exportfilename'), - ) - trades = history.trim_dataframe(trades, timerange, 'open_time') - return {"tickers": tickers, + no_trades = False + if config.get('no_trades', False): + no_trades = True + elif not config['exportfilename'].is_file() and config['trade_source'] == 'file': + logger.warning("Backtest file is missing skipping trades.") + no_trades = True + + trades = load_trades( + config['trade_source'], + db_url=config.get('db_url'), + exportfilename=config.get('exportfilename'), + no_trades=no_trades + ) + trades = trim_dataframe(trades, timerange, 'open_time') + + return {"ohlcv": data, "trades": trades, "pairs": pairs, } -def add_indicators(fig, row, indicators: List[str], data: pd.DataFrame) -> make_subplots: +def add_indicators(fig, row, indicators: Dict[str, Dict], data: pd.DataFrame) -> make_subplots: """ - Generator all the indicator selected by the user for a specific row + Generate all the indicators selected by the user for a specific row, based on the configuration :param fig: Plot figure to append to :param row: row number for this plot - :param indicators: List of indicators present in the dataframe + :param indicators: Dict of Indicators with configuration options. + Dict key must correspond to dataframe column. :param data: candlestick DataFrame """ - for indicator in indicators: + for indicator, conf in indicators.items(): + logger.debug(f"indicator {indicator} with config {conf}") if indicator in data: + kwargs = {'x': data['date'], + 'y': data[indicator].values, + 'mode': 'lines', + 'name': indicator + } + if 'color' in conf: + kwargs.update({'line': {'color': conf['color']}}) scatter = go.Scatter( - x=data['date'], - y=data[indicator].values, - mode='lines', - name=indicator + **kwargs ) fig.add_trace(scatter, row, 1) else: @@ -101,17 +123,68 @@ def add_profit(fig, row, data: pd.DataFrame, column: str, name: str) -> make_sub return fig +def add_max_drawdown(fig, row, trades: pd.DataFrame, df_comb: pd.DataFrame, + timeframe: str) -> make_subplots: + """ + Add scatter points indicating max drawdown + """ + try: + max_drawdown, highdate, lowdate = calculate_max_drawdown(trades) + + drawdown = go.Scatter( + x=[highdate, lowdate], + y=[ + df_comb.loc[timeframe_to_prev_date(timeframe, highdate), 'cum_profit'], + df_comb.loc[timeframe_to_prev_date(timeframe, lowdate), 'cum_profit'], + ], + mode='markers', + name=f"Max drawdown {max_drawdown * 100:.2f}%", + text=f"Max drawdown {max_drawdown * 100:.2f}%", + marker=dict( + symbol='square-open', + size=9, + line=dict(width=2), + color='green' + + ) + ) + fig.add_trace(drawdown, row, 1) + except ValueError: + logger.warning("No trades found - not plotting max drawdown.") + return fig + + def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: """ Add trades to "fig" """ # Trades can be empty if trades is not None and len(trades) > 0: + # Create description for sell summarizing the trade + trades['desc'] = trades.apply(lambda row: f"{round(row['profitperc'] * 100, 1)}%, " + f"{row['sell_reason']}, {row['duration']} min", + axis=1) trade_buys = go.Scatter( x=trades["open_time"], y=trades["open_rate"], mode='markers', - name='trade_buy', + name='Trade buy', + text=trades["desc"], + marker=dict( + symbol='circle-open', + size=11, + line=dict(width=2), + color='cyan' + + ) + ) + + trade_sells = go.Scatter( + x=trades.loc[trades['profitperc'] > 0, "close_time"], + y=trades.loc[trades['profitperc'] > 0, "close_rate"], + text=trades.loc[trades['profitperc'] > 0, "desc"], + mode='markers', + name='Sell - Profit', marker=dict( symbol='square-open', size=11, @@ -119,16 +192,12 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: color='green' ) ) - # Create description for sell summarizing the trade - desc = trades.apply(lambda row: f"{round(row['profitperc'], 3)}%, {row['sell_reason']}, " - f"{row['duration']} min", - axis=1) - trade_sells = go.Scatter( - x=trades["close_time"], - y=trades["close_rate"], - text=desc, + trade_sells_loss = go.Scatter( + x=trades.loc[trades['profitperc'] <= 0, "close_time"], + y=trades.loc[trades['profitperc'] <= 0, "close_rate"], + text=trades.loc[trades['profitperc'] <= 0, "desc"], mode='markers', - name='trade_sell', + name='Sell - Loss', marker=dict( symbol='square-open', size=11, @@ -138,14 +207,53 @@ def plot_trades(fig, trades: pd.DataFrame) -> make_subplots: ) fig.add_trace(trade_buys, 1, 1) fig.add_trace(trade_sells, 1, 1) + fig.add_trace(trade_sells_loss, 1, 1) else: logger.warning("No trades found.") return fig -def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFrame = None, +def create_plotconfig(indicators1: List[str], indicators2: List[str], + plot_config: Dict[str, Dict]) -> Dict[str, Dict]: + """ + Combines indicators 1 and indicators 2 into plot_config if necessary + :param indicators1: List containing Main plot indicators + :param indicators2: List containing Sub plot indicators + :param plot_config: Dict of Dicts containing advanced plot configuration + :return: plot_config - eventually with indicators 1 and 2 + """ + + if plot_config: + if indicators1: + plot_config['main_plot'] = {ind: {} for ind in indicators1} + if indicators2: + plot_config['subplots'] = {'Other': {ind: {} for ind in indicators2}} + + if not plot_config: + # If no indicators and no plot-config given, use defaults. + if not indicators1: + indicators1 = ['sma', 'ema3', 'ema5'] + if not indicators2: + indicators2 = ['macd', 'macdsignal'] + + # Create subplot configuration if plot_config is not available. + plot_config = { + 'main_plot': {ind: {} for ind in indicators1}, + 'subplots': {'Other': {ind: {} for ind in indicators2}}, + } + if 'main_plot' not in plot_config: + plot_config['main_plot'] = {} + + if 'subplots' not in plot_config: + plot_config['subplots'] = {} + return plot_config + + +def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFrame = None, *, indicators1: List[str] = [], - indicators2: List[str] = [],) -> go.Figure: + indicators2: List[str] = [], + plot_config: Dict[str, Dict] = {}, + ) -> go.Figure: """ Generate the graph from the data generated by Backtesting or from DB Volume will always be ploted in row2, so Row 1 and 3 are to our disposal for custom indicators @@ -154,21 +262,26 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra :param trades: All trades created :param indicators1: List containing Main plot indicators :param indicators2: List containing Sub plot indicators - :return: None + :param plot_config: Dict of Dicts containing advanced plot configuration + :return: Plotly figure """ + plot_config = create_plotconfig(indicators1, indicators2, plot_config) + rows = 2 + len(plot_config['subplots']) + row_widths = [1 for _ in plot_config['subplots']] # Define the graph fig = make_subplots( - rows=3, + rows=rows, cols=1, shared_xaxes=True, - row_width=[1, 1, 4], + row_width=row_widths + [1, 4], vertical_spacing=0.0001, ) fig['layout'].update(title=pair) fig['layout']['yaxis1'].update(title='Price') fig['layout']['yaxis2'].update(title='Volume') - fig['layout']['yaxis3'].update(title='Other') + for i, name in enumerate(plot_config['subplots']): + fig['layout'][f'yaxis{3 + i}'].update(title=name) fig['layout']['xaxis']['rangeslider'].update(visible=False) # Common information @@ -238,12 +351,13 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra ) fig.add_trace(bb_lower, 1, 1) fig.add_trace(bb_upper, 1, 1) - if 'bb_upperband' in indicators1 and 'bb_lowerband' in indicators1: - indicators1.remove('bb_upperband') - indicators1.remove('bb_lowerband') + if ('bb_upperband' in plot_config['main_plot'] + and 'bb_lowerband' in plot_config['main_plot']): + del plot_config['main_plot']['bb_upperband'] + del plot_config['main_plot']['bb_lowerband'] # Add indicators to main plot - fig = add_indicators(fig=fig, row=1, indicators=indicators1, data=data) + fig = add_indicators(fig=fig, row=1, indicators=plot_config['main_plot'], data=data) fig = plot_trades(fig, trades) @@ -254,19 +368,25 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra name='Volume', marker_color='DarkSlateGrey', marker_line_color='DarkSlateGrey' - ) + ) fig.add_trace(volume, 2, 1) # Add indicators to separate row - fig = add_indicators(fig=fig, row=3, indicators=indicators2, data=data) + for i, name in enumerate(plot_config['subplots']): + fig = add_indicators(fig=fig, row=3 + i, + indicators=plot_config['subplots'][name], + data=data) return fig -def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], +def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame], trades: pd.DataFrame, timeframe: str) -> go.Figure: # Combine close-values for all pairs, rename columns to "pair" - df_comb = combine_tickers_with_mean(tickers, "close") + df_comb = combine_dataframes_with_mean(data, "close") + + # Trim trades to available OHLCV data + trades = extract_trades_of_period(df_comb, trades, date_index=True) # Add combined cumulative profit df_comb = create_cum_profit(df_comb, trades, 'cum_profit', timeframe) @@ -290,6 +410,7 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], fig.add_trace(avgclose, 1, 1) fig = add_profit(fig, 2, df_comb, 'cum_profit', 'Profit') + fig = add_max_drawdown(fig, 2, trades, df_comb, timeframe) for pair in pairs: profit_col = f'cum_profit_{pair}' @@ -300,12 +421,12 @@ def generate_profit_graph(pairs: str, tickers: Dict[str, pd.DataFrame], return fig -def generate_plot_filename(pair, timeframe) -> str: +def generate_plot_filename(pair: str, timeframe: str) -> str: """ Generate filenames per pair/timeframe to be used for storing plots """ - pair_name = pair.replace("/", "_") - file_name = 'freqtrade-plot-' + pair_name + '-' + timeframe + '.html' + pair_s = pair_to_filename(pair) + file_name = 'freqtrade-plot-' + pair_s + '-' + timeframe + '.html' logger.info('Generate plot file for %s', pair) @@ -333,34 +454,33 @@ def load_and_plot_trades(config: Dict[str, Any]): """ From configuration provided - Initializes plot-script - - Get tickers data + - Get candle (OHLCV) data - Generate Dafaframes populated with indicators and signals based on configured strategy - Load trades excecuted during the selected period - Generate Plotly plot objects - Generate plot files :return: None """ - strategy = StrategyResolver(config).strategy + strategy = StrategyResolver.load_strategy(config) plot_elements = init_plotscript(config) trades = plot_elements['trades'] pair_counter = 0 - for pair, data in plot_elements["tickers"].items(): + for pair, data in plot_elements["ohlcv"].items(): pair_counter += 1 logger.info("analyse pair %s", pair) - tickers = {} - tickers[pair] = data - dataframe = strategy.analyze_ticker(tickers[pair], {'pair': pair}) + df_analyzed = strategy.analyze_ticker(data, {'pair': pair}) trades_pair = trades.loc[trades['pair'] == pair] - trades_pair = extract_trades_of_period(dataframe, trades_pair) + trades_pair = extract_trades_of_period(df_analyzed, trades_pair) fig = generate_candlestick_graph( pair=pair, - data=dataframe, + data=df_analyzed, trades=trades_pair, - indicators1=config["indicators1"], - indicators2=config["indicators2"], + indicators1=config.get("indicators1", []), + indicators2=config.get("indicators2", []), + plot_config=strategy.plot_config if hasattr(strategy, 'plot_config') else {} ) store_plot_file(fig, filename=generate_plot_filename(pair, config['ticker_interval']), @@ -387,7 +507,7 @@ def plot_profit(config: Dict[str, Any]) -> None: # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend - fig = generate_profit_graph(plot_elements["pairs"], plot_elements["tickers"], + fig = generate_profit_graph(plot_elements["pairs"], plot_elements["ohlcv"], trades, config.get('ticker_interval', '5m')) store_plot_file(fig, filename='freqtrade-profit-plot.html', directory=config['user_data_dir'] / "plot", auto_open=True) diff --git a/freqtrade/resolvers/exchange_resolver.py b/freqtrade/resolvers/exchange_resolver.py index 60f37b1c9..2b6a731a9 100644 --- a/freqtrade/resolvers/exchange_resolver.py +++ b/freqtrade/resolvers/exchange_resolver.py @@ -14,10 +14,10 @@ class ExchangeResolver(IResolver): """ This class contains all the logic to load a custom exchange class """ + object_type = Exchange - __slots__ = ['exchange'] - - def __init__(self, exchange_name: str, config: dict, validate: bool = True) -> None: + @staticmethod + def load_exchange(exchange_name: str, config: dict, validate: bool = True) -> Exchange: """ Load the custom class from config parameter :param config: configuration dictionary @@ -25,17 +25,20 @@ class ExchangeResolver(IResolver): # Map exchange name to avoid duplicate classes for identical exchanges exchange_name = MAP_EXCHANGE_CHILDCLASS.get(exchange_name, exchange_name) exchange_name = exchange_name.title() + exchange = None try: - self.exchange = self._load_exchange(exchange_name, kwargs={'config': config, - 'validate': validate}) + exchange = ExchangeResolver._load_exchange(exchange_name, + kwargs={'config': config, + 'validate': validate}) except ImportError: logger.info( f"No {exchange_name} specific subclass found. Using the generic class instead.") - if not hasattr(self, "exchange"): - self.exchange = Exchange(config, validate=validate) + if not exchange: + exchange = Exchange(config, validate=validate) + return exchange - def _load_exchange( - self, exchange_name: str, kwargs: dict) -> Exchange: + @staticmethod + def _load_exchange(exchange_name: str, kwargs: dict) -> Exchange: """ Loads the specified exchange. Only checks for exchanges exported in freqtrade.exchanges diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 05efa1164..ddf461252 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -5,10 +5,10 @@ This module load custom hyperopt """ import logging from pathlib import Path -from typing import Optional, Dict +from typing import Dict -from freqtrade import OperationalException from freqtrade.constants import DEFAULT_HYPEROPT_LOSS, USERPATH_HYPEROPTS +from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt_interface import IHyperOpt from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss from freqtrade.resolvers import IResolver @@ -20,11 +20,15 @@ class HyperOptResolver(IResolver): """ This class contains all the logic to load custom hyperopt class """ - __slots__ = ['hyperopt'] + object_type = IHyperOpt + object_type_str = "Hyperopt" + user_subdir = USERPATH_HYPEROPTS + initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - def __init__(self, config: Dict) -> None: + @staticmethod + def load_hyperopt(config: Dict) -> IHyperOpt: """ - Load the custom class from config parameter + Load the custom hyperopt class from config parameter :param config: configuration dictionary """ if not config.get('hyperopt'): @@ -33,50 +37,33 @@ class HyperOptResolver(IResolver): hyperopt_name = config['hyperopt'] - self.hyperopt = self._load_hyperopt(hyperopt_name, config, - extra_dir=config.get('hyperopt_path')) + hyperopt = HyperOptResolver.load_object(hyperopt_name, config, + kwargs={'config': config}, + extra_dir=config.get('hyperopt_path')) - if not hasattr(self.hyperopt, 'populate_indicators'): + if not hasattr(hyperopt, 'populate_indicators'): logger.warning("Hyperopt class does not provide populate_indicators() method. " "Using populate_indicators from the strategy.") - if not hasattr(self.hyperopt, 'populate_buy_trend'): + if not hasattr(hyperopt, 'populate_buy_trend'): logger.warning("Hyperopt class does not provide populate_buy_trend() method. " "Using populate_buy_trend from the strategy.") - if not hasattr(self.hyperopt, 'populate_sell_trend'): + if not hasattr(hyperopt, 'populate_sell_trend'): logger.warning("Hyperopt class does not provide populate_sell_trend() method. " "Using populate_sell_trend from the strategy.") - - def _load_hyperopt( - self, hyperopt_name: str, config: Dict, extra_dir: Optional[str] = None) -> IHyperOpt: - """ - Search and loads the specified hyperopt. - :param hyperopt_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given hyperopt - :return: HyperOpt instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, extra_dir=extra_dir) - - hyperopt = self._load_object(paths=abs_paths, object_type=IHyperOpt, - object_name=hyperopt_name, kwargs={'config': config}) - if hyperopt: - return hyperopt - raise OperationalException( - f"Impossible to load Hyperopt '{hyperopt_name}'. This class does not exist " - "or contains Python code errors." - ) + return hyperopt class HyperOptLossResolver(IResolver): """ This class contains all the logic to load custom hyperopt loss class """ - __slots__ = ['hyperoptloss'] + object_type = IHyperOptLoss + object_type_str = "HyperoptLoss" + user_subdir = USERPATH_HYPEROPTS + initial_search_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - def __init__(self, config: Dict) -> None: + @staticmethod + def load_hyperoptloss(config: Dict) -> IHyperOptLoss: """ Load the custom class from config parameter :param config: configuration dictionary @@ -86,38 +73,15 @@ class HyperOptLossResolver(IResolver): # default hyperopt loss hyperoptloss_name = config.get('hyperopt_loss') or DEFAULT_HYPEROPT_LOSS - self.hyperoptloss = self._load_hyperoptloss( - hyperoptloss_name, config, extra_dir=config.get('hyperopt_path')) + hyperoptloss = HyperOptLossResolver.load_object(hyperoptloss_name, + config, kwargs={}, + extra_dir=config.get('hyperopt_path')) # Assign ticker_interval to be used in hyperopt - self.hyperoptloss.__class__.ticker_interval = str(config['ticker_interval']) + hyperoptloss.__class__.ticker_interval = str(config['ticker_interval']) - if not hasattr(self.hyperoptloss, 'hyperopt_loss_function'): + if not hasattr(hyperoptloss, 'hyperopt_loss_function'): raise OperationalException( f"Found HyperoptLoss class {hyperoptloss_name} does not " "implement `hyperopt_loss_function`.") - - def _load_hyperoptloss( - self, hyper_loss_name: str, config: Dict, - extra_dir: Optional[str] = None) -> IHyperOptLoss: - """ - Search and loads the specified hyperopt loss class. - :param hyper_loss_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given hyperopt - :return: HyperOptLoss instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('optimize').resolve() - - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=USERPATH_HYPEROPTS, extra_dir=extra_dir) - - hyperoptloss = self._load_object(paths=abs_paths, object_type=IHyperOptLoss, - object_name=hyper_loss_name) - if hyperoptloss: - return hyperoptloss - - raise OperationalException( - f"Impossible to load HyperoptLoss '{hyper_loss_name}'. This class does not exist " - "or contains Python code errors." - ) + return hyperoptloss diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 3bad42fd9..52d944f2c 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -7,7 +7,9 @@ import importlib.util import inspect import logging from pathlib import Path -from typing import Any, List, Optional, Tuple, Union, Generator +from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union + +from freqtrade.exceptions import OperationalException logger = logging.getLogger(__name__) @@ -16,11 +18,19 @@ class IResolver: """ This class contains all the logic to load custom classes """ + # Childclasses need to override this + object_type: Type[Any] + object_type_str: str + user_subdir: Optional[str] = None + initial_search_path: Optional[Path] - def build_search_paths(self, config, current_path: Path, user_subdir: Optional[str] = None, + @classmethod + def build_search_paths(cls, config: Dict[str, Any], user_subdir: Optional[str] = None, extra_dir: Optional[str] = None) -> List[Path]: - abs_paths: List[Path] = [current_path] + abs_paths: List[Path] = [] + if cls.initial_search_path: + abs_paths.append(cls.initial_search_path) if user_subdir: abs_paths.insert(0, config['user_data_dir'].joinpath(user_subdir)) @@ -31,42 +41,47 @@ class IResolver: return abs_paths - @staticmethod - def _get_valid_object(object_type, module_path: Path, - object_name: str) -> Generator[Any, None, None]: + @classmethod + def _get_valid_object(cls, module_path: Path, object_name: Optional[str], + enum_failed: bool = False) -> Iterator[Any]: """ Generator returning objects with matching object_type and object_name in the path given. - :param object_type: object_type (class) :param module_path: absolute path to the module :param object_name: Class name of the object + :param enum_failed: If True, will return None for modules which fail. + Otherwise, failing modules are skipped. :return: generator containing matching objects """ # Generate spec based on absolute path # Pass object_name as first argument to have logging print a reasonable name. - spec = importlib.util.spec_from_file_location(object_name, str(module_path)) + spec = importlib.util.spec_from_file_location(object_name or "", str(module_path)) module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints except (ModuleNotFoundError, SyntaxError) as err: # Catch errors in case a specific module is not installed logger.warning(f"Could not import {module_path} due to '{err}'") + if enum_failed: + return iter([None]) valid_objects_gen = ( obj for name, obj in inspect.getmembers(module, inspect.isclass) - if object_name == name and object_type in obj.__bases__ + if ((object_name is None or object_name == name) and + issubclass(obj, cls.object_type) and obj is not cls.object_type) ) return valid_objects_gen - @staticmethod - def _search_object(directory: Path, object_type, object_name: str, - kwargs: dict = {}) -> Union[Tuple[Any, Path], Tuple[None, None]]: + @classmethod + def _search_object(cls, directory: Path, object_name: str + ) -> Union[Tuple[Any, Path], Tuple[None, None]]: """ Search for the objectname in the given directory :param directory: relative or absolute directory path - :return: object instance + :param object_name: ClassName of the object to load + :return: object class """ - logger.debug("Searching for %s %s in '%s'", object_type.__name__, object_name, directory) + logger.debug(f"Searching for {cls.object_type.__name__} {object_name} in '{directory}'") for entry in directory.iterdir(): # Only consider python files if not str(entry).endswith('.py'): @@ -74,14 +89,14 @@ class IResolver: continue module_path = entry.resolve() - obj = next(IResolver._get_valid_object(object_type, module_path, object_name), None) + obj = next(cls._get_valid_object(module_path, object_name), None) if obj: - return (obj(**kwargs), module_path) + return (obj, module_path) return (None, None) - @staticmethod - def _load_object(paths: List[Path], object_type, object_name: str, + @classmethod + def _load_object(cls, paths: List[Path], object_name: str, kwargs: dict = {}) -> Optional[Any]: """ Try to load object from path list. @@ -89,16 +104,67 @@ class IResolver: for _path in paths: try: - (module, module_path) = IResolver._search_object(directory=_path, - object_type=object_type, - object_name=object_name, - kwargs=kwargs) + (module, module_path) = cls._search_object(directory=_path, + object_name=object_name) if module: logger.info( - f"Using resolved {object_type.__name__.lower()[1:]} {object_name} " + f"Using resolved {cls.object_type.__name__.lower()[1:]} {object_name} " f"from '{module_path}'...") - return module + return module(**kwargs) except FileNotFoundError: logger.warning('Path "%s" does not exist.', _path.resolve()) return None + + @classmethod + def load_object(cls, object_name: str, config: dict, kwargs: dict, + extra_dir: Optional[str] = None) -> Any: + """ + Search and loads the specified object as configured in hte child class. + :param objectname: name of the module to import + :param config: configuration dictionary + :param extra_dir: additional directory to search for the given pairlist + :raises: OperationalException if the class is invalid or does not exist. + :return: Object instance or None + """ + + abs_paths = cls.build_search_paths(config, + user_subdir=cls.user_subdir, + extra_dir=extra_dir) + + pairlist = cls._load_object(paths=abs_paths, object_name=object_name, + kwargs=kwargs) + if pairlist: + return pairlist + raise OperationalException( + f"Impossible to load {cls.object_type_str} '{object_name}'. This class does not exist " + "or contains Python code errors." + ) + + @classmethod + def search_all_objects(cls, directory: Path, + enum_failed: bool) -> List[Dict[str, Any]]: + """ + Searches a directory for valid objects + :param directory: Path to search + :param enum_failed: If True, will return None for modules which fail. + Otherwise, failing modules are skipped. + :return: List of dicts containing 'name', 'class' and 'location' entires + """ + logger.debug(f"Searching for {cls.object_type.__name__} '{directory}'") + objects = [] + for entry in directory.iterdir(): + # Only consider python files + if not str(entry).endswith('.py'): + logger.debug('Ignoring %s', entry) + continue + module_path = entry.resolve() + logger.debug(f"Path {module_path}") + for obj in cls._get_valid_object(module_path, object_name=None, + enum_failed=enum_failed): + objects.append( + {'name': obj.__name__ if obj is not None else '', + 'class': obj, + 'location': entry, + }) + return objects diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index d849f4ffb..77db74084 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -6,7 +6,6 @@ This module load custom pairlists import logging from pathlib import Path -from freqtrade import OperationalException from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import IResolver @@ -17,41 +16,28 @@ class PairListResolver(IResolver): """ This class contains all the logic to load custom PairList class """ + object_type = IPairList + object_type_str = "Pairlist" + user_subdir = None + initial_search_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() - __slots__ = ['pairlist'] - - def __init__(self, pairlist_name: str, exchange, pairlistmanager, - config: dict, pairlistconfig: dict, pairlist_pos: int) -> None: + @staticmethod + def load_pairlist(pairlist_name: str, exchange, pairlistmanager, + config: dict, pairlistconfig: dict, pairlist_pos: int) -> IPairList: """ - Load the custom class from config parameter - :param config: configuration dictionary or None + Load the pairlist with pairlist_name + :param pairlist_name: Classname of the pairlist + :param exchange: Initialized exchange class + :param pairlistmanager: Initialized pairlist manager + :param config: configuration dictionary + :param pairlistconfig: Configuration dedicated to this pairlist + :param pairlist_pos: Position of the pairlist in the list of pairlists + :return: initialized Pairlist class """ - self.pairlist = self._load_pairlist(pairlist_name, config, + return PairListResolver.load_object(pairlist_name, config, kwargs={'exchange': exchange, 'pairlistmanager': pairlistmanager, 'config': config, 'pairlistconfig': pairlistconfig, - 'pairlist_pos': pairlist_pos}) - - def _load_pairlist( - self, pairlist_name: str, config: dict, kwargs: dict) -> IPairList: - """ - Search and loads the specified pairlist. - :param pairlist_name: name of the module to import - :param config: configuration dictionary - :param extra_dir: additional directory to search for the given pairlist - :return: PairList instance or None - """ - current_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() - - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=None, extra_dir=None) - - pairlist = self._load_object(paths=abs_paths, object_type=IPairList, - object_name=pairlist_name, kwargs=kwargs) - if pairlist: - return pairlist - raise OperationalException( - f"Impossible to load Pairlist '{pairlist_name}'. This class does not exist " - "or contains Python code errors." - ) + 'pairlist_pos': pairlist_pos}, + ) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 9a76b9b74..cddc7c9cd 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -9,9 +9,11 @@ from base64 import urlsafe_b64decode from collections import OrderedDict from inspect import getfullargspec from pathlib import Path -from typing import Dict, Optional +from typing import Any, Dict, Optional -from freqtrade import constants, OperationalException +from freqtrade.constants import (REQUIRED_ORDERTIF, REQUIRED_ORDERTYPES, + USERPATH_STRATEGIES) +from freqtrade.exceptions import OperationalException from freqtrade.resolvers import IResolver from freqtrade.strategy.interface import IStrategy @@ -20,12 +22,15 @@ logger = logging.getLogger(__name__) class StrategyResolver(IResolver): """ - This class contains all the logic to load custom strategy class + This class contains the logic to load custom strategy class """ + object_type = IStrategy + object_type_str = "Strategy" + user_subdir = USERPATH_STRATEGIES + initial_search_path = None - __slots__ = ['strategy'] - - def __init__(self, config: Optional[Dict] = None) -> None: + @staticmethod + def load_strategy(config: Dict[str, Any] = None) -> IStrategy: """ Load the custom class from config parameter :param config: configuration dictionary or None @@ -37,9 +42,9 @@ class StrategyResolver(IResolver): "the strategy class to use.") strategy_name = config['strategy'] - self.strategy: IStrategy = self._load_strategy(strategy_name, - config=config, - extra_dir=config.get('strategy_path')) + strategy: IStrategy = StrategyResolver._load_strategy( + strategy_name, config=config, + extra_dir=config.get('strategy_path')) # make sure ask_strategy dict is available if 'ask_strategy' not in config: @@ -61,15 +66,18 @@ class StrategyResolver(IResolver): ("stake_currency", None, False), ("stake_amount", None, False), ("startup_candle_count", None, False), + ("unfilledtimeout", None, False), ("use_sell_signal", True, True), ("sell_profit_only", False, True), ("ignore_roi_if_buy_signal", False, True), ] for attribute, default, ask_strategy in attributes: if ask_strategy: - self._override_attribute_helper(config['ask_strategy'], attribute, default) + StrategyResolver._override_attribute_helper(strategy, config['ask_strategy'], + attribute, default) else: - self._override_attribute_helper(config, attribute, default) + StrategyResolver._override_attribute_helper(strategy, config, + attribute, default) # Loop this list again to have output combined for attribute, _, exp in attributes: @@ -79,14 +87,17 @@ class StrategyResolver(IResolver): logger.info("Strategy using %s: %s", attribute, config[attribute]) # Sort and apply type conversions - self.strategy.minimal_roi = OrderedDict(sorted( - {int(key): value for (key, value) in self.strategy.minimal_roi.items()}.items(), + strategy.minimal_roi = OrderedDict(sorted( + {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), key=lambda t: t[0])) - self.strategy.stoploss = float(self.strategy.stoploss) + strategy.stoploss = float(strategy.stoploss) - self._strategy_sanity_validations() + StrategyResolver._strategy_sanity_validations(strategy) + return strategy - def _override_attribute_helper(self, config, attribute: str, default): + @staticmethod + def _override_attribute_helper(strategy, config: Dict[str, Any], + attribute: str, default: Any): """ Override attributes in the strategy. Prevalence: @@ -95,30 +106,32 @@ class StrategyResolver(IResolver): - default (if not None) """ if attribute in config: - setattr(self.strategy, attribute, config[attribute]) + setattr(strategy, attribute, config[attribute]) logger.info("Override strategy '%s' with value in config file: %s.", attribute, config[attribute]) - elif hasattr(self.strategy, attribute): - val = getattr(self.strategy, attribute) + elif hasattr(strategy, attribute): + val = getattr(strategy, attribute) # None's cannot exist in the config, so do not copy them if val is not None: config[attribute] = val # Explicitly check for None here as other "falsy" values are possible elif default is not None: - setattr(self.strategy, attribute, default) + setattr(strategy, attribute, default) config[attribute] = default - def _strategy_sanity_validations(self): - if not all(k in self.strategy.order_types for k in constants.REQUIRED_ORDERTYPES): - raise ImportError(f"Impossible to load Strategy '{self.strategy.__class__.__name__}'. " + @staticmethod + def _strategy_sanity_validations(strategy): + if not all(k in strategy.order_types for k in REQUIRED_ORDERTYPES): + raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. " f"Order-types mapping is incomplete.") - if not all(k in self.strategy.order_time_in_force for k in constants.REQUIRED_ORDERTIF): - raise ImportError(f"Impossible to load Strategy '{self.strategy.__class__.__name__}'. " + if not all(k in strategy.order_time_in_force for k in REQUIRED_ORDERTIF): + raise ImportError(f"Impossible to load Strategy '{strategy.__class__.__name__}'. " f"Order-time-in-force mapping is incomplete.") - def _load_strategy( - self, strategy_name: str, config: dict, extra_dir: Optional[str] = None) -> IStrategy: + @staticmethod + def _load_strategy(strategy_name: str, + config: dict, extra_dir: Optional[str] = None) -> IStrategy: """ Search and loads the specified strategy. :param strategy_name: name of the module to import @@ -126,11 +139,10 @@ class StrategyResolver(IResolver): :param extra_dir: additional directory to search for the given strategy :return: Strategy instance or None """ - current_path = Path(__file__).parent.parent.joinpath('strategy').resolve() - abs_paths = self.build_search_paths(config, current_path=current_path, - user_subdir=constants.USERPATH_STRATEGY, - extra_dir=extra_dir) + abs_paths = StrategyResolver.build_search_paths(config, + user_subdir=USERPATH_STRATEGIES, + extra_dir=extra_dir) if ":" in strategy_name: logger.info("loading base64 encoded strategy") @@ -148,8 +160,9 @@ class StrategyResolver(IResolver): # register temp path with the bot abs_paths.insert(0, temp.resolve()) - strategy = self._load_object(paths=abs_paths, object_type=IStrategy, - object_name=strategy_name, kwargs={'config': config}) + strategy = StrategyResolver._load_object(paths=abs_paths, + object_name=strategy_name, + kwargs={'config': config}) if strategy: strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args) strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args) diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 8f4cc4787..61eacf639 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -2,11 +2,17 @@ import logging import threading from datetime import date, datetime from ipaddress import IPv4Address -from typing import Dict, Callable, Any +from typing import Any, Callable, Dict from arrow import Arrow from flask import Flask, jsonify, request from flask.json import JSONEncoder +from flask_cors import CORS +from flask_jwt_extended import (JWTManager, create_access_token, + create_refresh_token, get_jwt_identity, + jwt_refresh_token_required, + verify_jwt_in_request_optional) +from werkzeug.security import safe_str_cmp from werkzeug.serving import make_server from freqtrade.__init__ import __version__ @@ -38,9 +44,9 @@ class ArrowJSONEncoder(JSONEncoder): def require_login(func: Callable[[Any, Any], Any]): def func_wrapper(obj, *args, **kwargs): - + verify_jwt_in_request_optional() auth = request.authorization - if auth and obj.check_auth(auth.username, auth.password): + if get_jwt_identity() or auth and obj.check_auth(auth.username, auth.password): return func(obj, *args, **kwargs) else: return jsonify({"error": "Unauthorized"}), 401 @@ -70,8 +76,8 @@ class ApiServer(RPC): """ def check_auth(self, username, password): - return (username == self._config['api_server'].get('username') and - password == self._config['api_server'].get('password')) + return (safe_str_cmp(username, self._config['api_server'].get('username')) and + safe_str_cmp(password, self._config['api_server'].get('password'))) def __init__(self, freqtrade) -> None: """ @@ -83,6 +89,13 @@ class ApiServer(RPC): self._config = freqtrade.config self.app = Flask(__name__) + self._cors = CORS(self.app, resources={r"/api/*": {"origins": "*"}}) + + # Setup the Flask-JWT-Extended extension + self.app.config['JWT_SECRET_KEY'] = self._config['api_server'].get( + 'jwt_secret_key', 'super-secret') + + self.jwt = JWTManager(self.app) self.app.json_encoder = ArrowJSONEncoder # Register application handling @@ -148,6 +161,10 @@ class ApiServer(RPC): self.app.register_error_handler(404, self.page_not_found) # Actions to control the bot + self.app.add_url_rule(f'{BASE_URI}/token/login', 'login', + view_func=self._token_login, methods=['POST']) + self.app.add_url_rule(f'{BASE_URI}/token/refresh', 'token_refresh', + view_func=self._token_refresh, methods=['POST']) self.app.add_url_rule(f'{BASE_URI}/start', 'start', view_func=self._start, methods=['POST']) self.app.add_url_rule(f'{BASE_URI}/stop', 'stop', view_func=self._stop, methods=['POST']) @@ -173,7 +190,8 @@ class ApiServer(RPC): view_func=self._show_config, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/ping', 'ping', view_func=self._ping, methods=['GET']) - + self.app.add_url_rule(f'{BASE_URI}/trades', 'trades', + view_func=self._trades, methods=['GET']) # Combined actions and infos self.app.add_url_rule(f'{BASE_URI}/blacklist', 'blacklist', view_func=self._blacklist, methods=['GET', 'POST']) @@ -198,6 +216,37 @@ class ApiServer(RPC): 'code': 404 }), 404 + @require_login + @rpc_catch_errors + def _token_login(self): + """ + Handler for /token/login + Returns a JWT token + """ + auth = request.authorization + if auth and self.check_auth(auth.username, auth.password): + keystuff = {'u': auth.username} + ret = { + 'access_token': create_access_token(identity=keystuff), + 'refresh_token': create_refresh_token(identity=keystuff), + } + return self.rest_dump(ret) + + return jsonify({"error": "Unauthorized"}), 401 + + @jwt_refresh_token_required + @rpc_catch_errors + def _token_refresh(self): + """ + Handler for /token/refresh + Returns a JWT token based on a JWT refresh token + """ + current_user = get_jwt_identity() + new_token = create_access_token(identity=current_user, fresh=False) + + ret = {'access_token': new_token} + return self.rest_dump(ret) + @require_login @rpc_catch_errors def _start(self): @@ -358,6 +407,18 @@ class ApiServer(RPC): self._config.get('fiat_display_currency', '')) return self.rest_dump(results) + @require_login + @rpc_catch_errors + def _trades(self): + """ + Handler for /trades. + + Returns the X last trades in json format + """ + limit = int(request.args.get('limit', 0)) + results = self._rpc_trade_history(limit) + return self.rest_dump(results) + @require_login @rpc_catch_errors def _whitelist(self): diff --git a/freqtrade/rpc/fiat_convert.py b/freqtrade/rpc/fiat_convert.py index d40f9221e..4e26432d4 100644 --- a/freqtrade/rpc/fiat_convert.py +++ b/freqtrade/rpc/fiat_convert.py @@ -7,7 +7,7 @@ import logging import time from typing import Dict, List -from coinmarketcap import Market +from pycoingecko import CoinGeckoAPI from freqtrade.constants import SUPPORTED_FIAT @@ -38,8 +38,8 @@ class CryptoFiat: # Private attributes self._expiration = 0.0 - self.crypto_symbol = crypto_symbol.upper() - self.fiat_symbol = fiat_symbol.upper() + self.crypto_symbol = crypto_symbol.lower() + self.fiat_symbol = fiat_symbol.lower() self.set_price(price=price) def set_price(self, price: float) -> None: @@ -67,17 +67,20 @@ class CryptoToFiatConverter: This object is also a Singleton """ __instance = None - _coinmarketcap: Market = None + _coingekko: CoinGeckoAPI = None _cryptomap: Dict = {} def __new__(cls): + """ + This class is a singleton - cannot be instantiated twice. + """ if CryptoToFiatConverter.__instance is None: CryptoToFiatConverter.__instance = object.__new__(cls) try: - CryptoToFiatConverter._coinmarketcap = Market() + CryptoToFiatConverter._coingekko = CoinGeckoAPI() except BaseException: - CryptoToFiatConverter._coinmarketcap = None + CryptoToFiatConverter._coingekko = None return CryptoToFiatConverter.__instance def __init__(self) -> None: @@ -86,14 +89,12 @@ class CryptoToFiatConverter: def _load_cryptomap(self) -> None: try: - coinlistings = self._coinmarketcap.listings() - self._cryptomap = dict(map(lambda coin: (coin["symbol"], str(coin["id"])), - coinlistings["data"])) - except (BaseException) as exception: + coinlistings = self._coingekko.get_coins_list() + # Create mapping table from synbol to coingekko_id + self._cryptomap = {x['symbol']: x['id'] for x in coinlistings} + except (Exception) as exception: logger.error( - "Could not load FIAT Cryptocurrency map for the following problem: %s", - type(exception).__name__ - ) + f"Could not load FIAT Cryptocurrency map for the following problem: {exception}") def convert_amount(self, crypto_amount: float, crypto_symbol: str, fiat_symbol: str) -> float: """ @@ -115,8 +116,8 @@ class CryptoToFiatConverter: :param fiat_symbol: FIAT currency you want to convert to (e.g USD) :return: Price in FIAT """ - crypto_symbol = crypto_symbol.upper() - fiat_symbol = fiat_symbol.upper() + crypto_symbol = crypto_symbol.lower() + fiat_symbol = fiat_symbol.lower() # Check if the fiat convertion you want is supported if not self._is_supported_fiat(fiat=fiat_symbol): @@ -170,15 +171,13 @@ class CryptoToFiatConverter: :return: bool, True supported, False not supported """ - fiat = fiat.upper() - - return fiat in SUPPORTED_FIAT + return fiat.upper() in SUPPORTED_FIAT def _find_price(self, crypto_symbol: str, fiat_symbol: str) -> float: """ - Call CoinMarketCap API to retrieve the price in the FIAT - :param crypto_symbol: Crypto-currency you want to convert (e.g BTC) - :param fiat_symbol: FIAT currency you want to convert to (e.g USD) + Call CoinGekko API to retrieve the price in the FIAT + :param crypto_symbol: Crypto-currency you want to convert (e.g btc) + :param fiat_symbol: FIAT currency you want to convert to (e.g usd) :return: float, price of the crypto-currency in Fiat """ # Check if the fiat convertion you want is supported @@ -195,12 +194,13 @@ class CryptoToFiatConverter: return 0.0 try: + _gekko_id = self._cryptomap[crypto_symbol] return float( - self._coinmarketcap.ticker( - currency=self._cryptomap[crypto_symbol], - convert=fiat_symbol - )['data']['quotes'][fiat_symbol.upper()]['price'] + self._coingekko.get_price( + ids=_gekko_id, + vs_currencies=fiat_symbol + )[_gekko_id][fiat_symbol] ) - except BaseException as exception: + except Exception as exception: logger.error("Error in _find_price: %s", exception) return 0.0 diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 3b4b7570a..21f54de50 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple import arrow from numpy import NAN, mean -from freqtrade import DependencyException, TemporaryError +from freqtrade.exceptions import DependencyException, TemporaryError from freqtrade.misc import shorten_date from freqtrade.persistence import Trade from freqtrade.rpc.fiat_convert import CryptoToFiatConverter @@ -26,7 +26,9 @@ class RPCMessageType(Enum): WARNING_NOTIFICATION = 'warning' CUSTOM_NOTIFICATION = 'custom' BUY_NOTIFICATION = 'buy' + BUY_CANCEL_NOTIFICATION = 'buy_cancel' SELL_NOTIFICATION = 'sell' + SELL_CANCEL_NOTIFICATION = 'sell_cancel' def __repr__(self): return self.value @@ -39,6 +41,7 @@ class RPCException(Exception): raise RPCException('*Status:* `no active trade`') """ + def __init__(self, message: str) -> None: super().__init__(self) self.message = message @@ -88,9 +91,10 @@ class RPC: """ config = self._freqtrade.config val = { - 'dry_run': config.get('dry_run', False), + 'dry_run': config['dry_run'], 'stake_currency': config['stake_currency'], 'stake_amount': config['stake_amount'], + 'max_open_trades': config['max_open_trades'], 'minimal_roi': config['minimal_roi'].copy(), 'stoploss': config['stoploss'], 'trailing_stop': config['trailing_stop'], @@ -100,6 +104,8 @@ class RPC: 'ticker_interval': config['ticker_interval'], 'exchange': config['exchange']['name'], 'strategy': config['strategy'], + 'forcebuy_enabled': config.get('forcebuy_enable', False), + 'state': str(self._freqtrade.state) } return val @@ -139,10 +145,11 @@ class RPC: results.append(trade_dict) return results - def _rpc_status_table(self, stake_currency, fiat_display_currency: str) -> Tuple[List, List]: + def _rpc_status_table(self, stake_currency: str, + fiat_display_currency: str) -> Tuple[List, List]: trades = Trade.get_open_trades() if not trades: - raise RPCException('no active order') + raise RPCException('no active trade') else: trades_list = [] for trade in trades: @@ -151,20 +158,22 @@ class RPC: current_rate = self._freqtrade.get_sell_rate(trade.pair, False) except DependencyException: current_rate = NAN - trade_perc = (100 * trade.calc_profit_ratio(current_rate)) + trade_percent = (100 * trade.calc_profit_ratio(current_rate)) trade_profit = trade.calc_profit(current_rate) - profit_str = f'{trade_perc:.2f}%' + profit_str = f'{trade_percent:.2f}%' if self._fiat_converter: fiat_profit = self._fiat_converter.convert_amount( - trade_profit, - stake_currency, - fiat_display_currency - ) + trade_profit, + stake_currency, + fiat_display_currency + ) if fiat_profit and not isnan(fiat_profit): profit_str += f" ({fiat_profit:.2f})" trades_list.append([ trade.id, - trade.pair, + trade.pair + ('*' if (trade.open_order_id is not None + and trade.close_rate_requested is None) else '') + + ('**' if (trade.close_rate_requested is not None) else ''), shorten_date(arrow.get(trade.open_date).humanize(only_distance=True)), profit_str ]) @@ -177,7 +186,7 @@ class RPC: def _rpc_daily_profit( self, timescale: int, - stake_currency: str, fiat_display_currency: str) -> List[List[Any]]: + stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: today = datetime.utcnow().date() profit_days: Dict[date, Dict] = {} @@ -191,34 +200,46 @@ class RPC: Trade.close_date >= profitday, Trade.close_date < (profitday + timedelta(days=1)) ]).order_by(Trade.close_date).all() - curdayprofit = sum(trade.calc_profit() for trade in trades) + curdayprofit = sum(trade.close_profit_abs for trade in trades) profit_days[profitday] = { 'amount': f'{curdayprofit:.8f}', 'trades': len(trades) } - return [ - [ - key, - '{value:.8f} {symbol}'.format( - value=float(value['amount']), - symbol=stake_currency - ), - '{value:.3f} {symbol}'.format( + data = [ + { + 'date': key, + 'abs_profit': f'{float(value["amount"]):.8f}', + 'fiat_value': '{value:.3f}'.format( value=self._fiat_converter.convert_amount( value['amount'], stake_currency, fiat_display_currency ) if self._fiat_converter else 0, - symbol=fiat_display_currency ), - '{value} trade{s}'.format( - value=value['trades'], - s='' if value['trades'] < 2 else 's' - ), - ] + 'trade_count': f'{value["trades"]}', + } for key, value in profit_days.items() ] + return { + 'stake_currency': stake_currency, + 'fiat_display_currency': fiat_display_currency, + 'data': data + } + + def _rpc_trade_history(self, limit: int) -> Dict: + """ Returns the X last trades """ + if limit > 0: + trades = Trade.get_trades().order_by(Trade.id.desc()).limit(limit) + else: + trades = Trade.get_trades().order_by(Trade.id.desc()).all() + + output = [trade.to_json() for trade in trades] + + return { + "trades": output, + "trades_count": len(output) + } def _rpc_trade_statistics( self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: @@ -226,9 +247,9 @@ class RPC: trades = Trade.get_trades().order_by(Trade.id).all() profit_all_coin = [] - profit_all_perc = [] + profit_all_ratio = [] profit_closed_coin = [] - profit_closed_perc = [] + profit_closed_ratio = [] durations = [] for trade in trades: @@ -240,21 +261,21 @@ class RPC: durations.append((trade.close_date - trade.open_date).total_seconds()) if not trade.is_open: - profit_percent = trade.calc_profit_ratio() - profit_closed_coin.append(trade.calc_profit()) - profit_closed_perc.append(profit_percent) + profit_ratio = trade.close_profit + profit_closed_coin.append(trade.close_profit_abs) + profit_closed_ratio.append(profit_ratio) else: # Get current rate try: current_rate = self._freqtrade.get_sell_rate(trade.pair, False) except DependencyException: current_rate = NAN - profit_percent = trade.calc_profit_ratio(rate=current_rate) + profit_ratio = trade.calc_profit_ratio(rate=current_rate) profit_all_coin.append( trade.calc_profit(rate=trade.close_rate or current_rate) ) - profit_all_perc.append(profit_percent) + profit_all_ratio.append(profit_ratio) best_pair = Trade.get_best_pair() @@ -265,7 +286,7 @@ class RPC: # Prepare data to display profit_closed_coin_sum = round(sum(profit_closed_coin), 8) - profit_closed_percent = (round(mean(profit_closed_perc) * 100, 2) if profit_closed_perc + profit_closed_percent = (round(mean(profit_closed_ratio) * 100, 2) if profit_closed_ratio else 0.0) profit_closed_fiat = self._fiat_converter.convert_amount( profit_closed_coin_sum, @@ -274,7 +295,7 @@ class RPC: ) if self._fiat_converter else 0 profit_all_coin_sum = round(sum(profit_all_coin), 8) - profit_all_percent = round(mean(profit_all_perc) * 100, 2) if profit_all_perc else 0.0 + profit_all_percent = round(mean(profit_all_ratio) * 100, 2) if profit_all_ratio else 0.0 profit_all_fiat = self._fiat_converter.convert_amount( profit_all_coin_sum, stake_currency, @@ -306,6 +327,8 @@ class RPC: except (TemporaryError, DependencyException): raise RPCException('Error getting current tickers.') + self._freqtrade.wallets.update(require_update=False) + for coin, balance in self._freqtrade.wallets.get_all_balances().items(): if not balance.total: continue @@ -335,20 +358,21 @@ class RPC: 'stake': stake_currency, }) if total == 0.0: - if self._freqtrade.config.get('dry_run', False): + if self._freqtrade.config['dry_run']: raise RPCException('Running in Dry Run, balances are not available.') else: raise RPCException('All balances are zero.') symbol = fiat_display_currency - value = self._fiat_converter.convert_amount(total, 'BTC', + value = self._fiat_converter.convert_amount(total, stake_currency, symbol) if self._fiat_converter else 0 return { 'currencies': output, 'total': total, 'symbol': symbol, 'value': value, - 'note': 'Simulated balances' if self._freqtrade.config.get('dry_run', False) else '' + 'stake': stake_currency, + 'note': 'Simulated balances' if self._freqtrade.config['dry_run'] else '' } def _rpc_start(self) -> Dict[str, str]: @@ -382,7 +406,7 @@ class RPC: return {'status': 'No more buy will occur from now. Run /reload_conf to reset.'} - def _rpc_forcesell(self, trade_id) -> Dict[str, str]: + def _rpc_forcesell(self, trade_id: str) -> Dict[str, str]: """ Handler for forcesell . Sells the given trade at current price @@ -417,24 +441,27 @@ class RPC: if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') - if trade_id == 'all': - # Execute sell for all open orders - for trade in Trade.get_open_trades(): - _exec_forcesell(trade) + with self._freqtrade._sell_lock: + if trade_id == 'all': + # Execute sell for all open orders + for trade in Trade.get_open_trades(): + _exec_forcesell(trade) + Trade.session.flush() + self._freqtrade.wallets.update() + return {'result': 'Created sell orders for all open trades.'} + + # Query for trade + trade = Trade.get_trades( + trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ] + ).first() + if not trade: + logger.warning('forcesell: Invalid argument received') + raise RPCException('invalid argument') + + _exec_forcesell(trade) Trade.session.flush() - return {'result': 'Created sell orders for all open trades.'} - - # Query for trade - trade = Trade.get_trades( - trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True), ] - ).first() - if not trade: - logger.warning('forcesell: Invalid argument received') - raise RPCException('invalid argument') - - _exec_forcesell(trade) - Trade.session.flush() - return {'result': f'Created sell order for trade {trade_id}.'} + self._freqtrade.wallets.update() + return {'result': f'Created sell order for trade {trade_id}.'} def _rpc_forcebuy(self, pair: str, price: Optional[float]) -> Optional[Trade]: """ @@ -448,9 +475,9 @@ class RPC: if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') - # Check pair is in stake currency + # Check if pair quote currency equals to the stake currency. stake_currency = self._freqtrade.config.get('stake_currency') - if not pair.endswith(stake_currency): + if not self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency: raise RPCException( f'Wrong pair selected. Please pairs with stake {stake_currency} pairs only') # check if valid pair @@ -461,7 +488,7 @@ class RPC: raise RPCException(f'position for {pair} already open - id: {trade.id}') # gen stake amount - stakeamount = self._freqtrade._get_trade_stake_amount(pair) + stakeamount = self._freqtrade.get_trade_stake_amount(pair) # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price): @@ -505,7 +532,7 @@ class RPC: if add: stake_currency = self._freqtrade.config.get('stake_currency') for pair in add: - if (pair.endswith(stake_currency) + if (self._freqtrade.exchange.get_pair_quote_currency(pair) == stake_currency and pair not in self._freqtrade.pairlists.blacklist): self._freqtrade.pairlists.blacklist.append(pair) @@ -518,5 +545,5 @@ class RPC: def _rpc_edge(self) -> List[Dict[str, Any]]: """ Returns information related to Edge """ if not self._freqtrade.edge: - raise RPCException(f'Edge is not enabled.') + raise RPCException('Edge is not enabled.') return self._freqtrade.edge.accepted_pairs() diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index cb9e697e9..670275991 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -61,8 +61,8 @@ class RPCManager: except NotImplementedError: logger.error(f"Message type {msg['type']} not implemented by handler {mod.name}.") - def startup_messages(self, config, pairlist) -> None: - if config.get('dry_run', False): + def startup_messages(self, config: Dict[str, Any], pairlist) -> None: + if config['dry_run']: self.send_msg({ 'type': RPCMessageType.WARNING_NOTIFICATION, 'status': 'Dry run is enabled. All trades are simulated.' diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index e0e2afd7b..dfda15a26 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -134,25 +134,30 @@ class Telegram(RPC): msg['stake_amount_fiat'] = 0 message = ("*{exchange}:* Buying {pair}\n" - "at rate `{limit:.8f}\n" - "({stake_amount:.6f} {stake_currency}").format(**msg) + "*Amount:* `{amount:.8f}`\n" + "*Open Rate:* `{limit:.8f}`\n" + "*Current Rate:* `{current_rate:.8f}`\n" + "*Total:* `({stake_amount:.6f} {stake_currency}").format(**msg) if msg.get('fiat_currency', None): - message += ",{stake_amount_fiat:.3f} {fiat_currency}".format(**msg) + message += ", {stake_amount_fiat:.3f} {fiat_currency}".format(**msg) message += ")`" + elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: + message = "*{exchange}:* Cancelling Open Buy Order for {pair}".format(**msg) + elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: msg['amount'] = round(msg['amount'], 8) - msg['profit_percent'] = round(msg['profit_percent'] * 100, 2) + msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2) msg['duration'] = msg['close_date'].replace( microsecond=0) - msg['open_date'].replace(microsecond=0) msg['duration_min'] = msg['duration'].total_seconds() / 60 message = ("*{exchange}:* Selling {pair}\n" - "*Rate:* `{limit:.8f}`\n" "*Amount:* `{amount:.8f}`\n" "*Open Rate:* `{open_rate:.8f}`\n" "*Current Rate:* `{current_rate:.8f}`\n" + "*Close Rate:* `{limit:.8f}`\n" "*Sell Reason:* `{sell_reason}`\n" "*Duration:* `{duration} ({duration_min:.1f} min)`\n" "*Profit:* `{profit_percent:.2f}%`").format(**msg) @@ -163,8 +168,12 @@ class Telegram(RPC): and self._fiat_converter): msg['profit_fiat'] = self._fiat_converter.convert_amount( msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) - message += ('` ({gain}: {profit_amount:.8f} {stake_currency}`' - '` / {profit_fiat:.3f} {fiat_currency})`').format(**msg) + message += (' `({gain}: {profit_amount:.8f} {stake_currency}' + ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg) + + elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION: + message = ("*{exchange}:* Cancelling Open Sell Order " + "for {pair}. Reason: {reason}").format(**msg) elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION: message = '*Status:* `{status}`'.format(**msg) @@ -217,11 +226,15 @@ class Telegram(RPC): # Adding stoploss and stoploss percentage only if it is not None "*Stoploss:* `{stop_loss:.8f}` " + ("`({stop_loss_pct:.2f}%)`" if r['stop_loss_pct'] else ""), - - "*Open Order:* `{open_order}`" if r['open_order'] else "" ] + if r['open_order']: + if r['sell_order_status']: + lines.append("*Open Order:* `{open_order}` - `{sell_order_status}`") + else: + lines.append("*Open Order:* `{open_order}`") + # Filter empty lines using list-comprehension - messages.append("\n".join([l for l in lines if l]).format(**r)) + messages.append("\n".join([line for line in lines if line]).format(**r)) for msg in messages: self._send_msg(msg) @@ -267,14 +280,18 @@ class Telegram(RPC): stake_cur, fiat_disp_cur ) - stats_tab = tabulate(stats, - headers=[ - 'Day', - f'Profit {stake_cur}', - f'Profit {fiat_disp_cur}', - f'Trades' - ], - tablefmt='simple') + stats_tab = tabulate( + [[day['date'], + f"{day['abs_profit']} {stats['stake_currency']}", + f"{day['fiat_value']} {stats['fiat_display_currency']}", + f"{day['trade_count']} trades"] for day in stats['data']], + headers=[ + 'Day', + f'Profit {stake_cur}', + f'Profit {fiat_disp_cur}', + 'Trades', + ], + tablefmt='simple') message = f'Daily Profit over the last {timescale} days:\n
{stats_tab}
' self._send_msg(message, parse_mode=ParseMode.HTML) except RPCException as e: @@ -335,7 +352,7 @@ class Telegram(RPC): output = '' if self._config['dry_run']: output += ( - f"*Warning:*Simulated balances in Dry Mode.\n" + f"*Warning:* Simulated balances in Dry Mode.\n" "This mode is still experimental!\n" "Starting capital: " f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" @@ -358,7 +375,7 @@ class Telegram(RPC): output += curr_output output += "\n*Estimated Value*:\n" \ - "\t`BTC: {total: .8f}`\n" \ + "\t`{stake}: {total: .8f}`\n" \ "\t`{symbol}: {value: .2f}`\n".format(**result) self._send_msg(output) except RPCException as e: @@ -553,6 +570,8 @@ class Telegram(RPC): "*/stop:* `Stops the trader`\n" \ "*/status [table]:* `Lists all open trades`\n" \ " *table :* `will display trades in a table`\n" \ + " `pending buy orders are marked with an asterisk (*)`\n" \ + " `pending sell orders are marked with a double asterisk (**)`\n" \ "*/profit:* `Lists cumulative profit from all finished trades`\n" \ "*/forcesell |all:* `Instantly sells the given trade or all trades, " \ "regardless of profit`\n" \ @@ -568,7 +587,7 @@ class Telegram(RPC): "*/whitelist:* `Show current whitelist` \n" \ "*/blacklist [pair]:* `Show current blacklist, or adds one or more pairs " \ "to the blacklist.` \n" \ - "*/edge:* `Shows validated pairs by Edge if it is enabeld` \n" \ + "*/edge:* `Shows validated pairs by Edge if it is enabled` \n" \ "*/help:* `This help message`\n" \ "*/version:* `Show version`" @@ -610,10 +629,12 @@ class Telegram(RPC): f"*Mode:* `{'Dry-run' if val['dry_run'] else 'Live'}`\n" f"*Exchange:* `{val['exchange']}`\n" f"*Stake per trade:* `{val['stake_amount']} {val['stake_currency']}`\n" + f"*Max open Trades:* `{val['max_open_trades']}`\n" f"*Minimum ROI:* `{val['minimal_roi']}`\n" f"{sl_info}" f"*Ticker Interval:* `{val['ticker_interval']}`\n" - f"*Strategy:* `{val['strategy']}`" + f"*Strategy:* `{val['strategy']}`\n" + f"*Current state:* `{val['state']}`" ) def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN) -> None: diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 37ca466de..322d990ee 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -41,11 +41,15 @@ class Webhook(RPC): if msg['type'] == RPCMessageType.BUY_NOTIFICATION: valuedict = self._config['webhook'].get('webhookbuy', None) + elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION: + valuedict = self._config['webhook'].get('webhookbuycancel', None) elif msg['type'] == RPCMessageType.SELL_NOTIFICATION: valuedict = self._config['webhook'].get('webhooksell', None) - elif msg['type'] in(RPCMessageType.STATUS_NOTIFICATION, - RPCMessageType.CUSTOM_NOTIFICATION, - RPCMessageType.WARNING_NOTIFICATION): + elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION: + valuedict = self._config['webhook'].get('webhooksellcancel', None) + elif msg['type'] in (RPCMessageType.STATUS_NOTIFICATION, + RPCMessageType.CUSTOM_NOTIFICATION, + RPCMessageType.WARNING_NOTIFICATION): valuedict = self._config['webhook'].get('webhookstatus', None) else: raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) diff --git a/freqtrade/state.py b/freqtrade/state.py index 415f6f5f2..38784c6a4 100644 --- a/freqtrade/state.py +++ b/freqtrade/state.py @@ -14,6 +14,9 @@ class State(Enum): STOPPED = 2 RELOAD_CONF = 3 + def __str__(self): + return f"{self.name.lower()}" + class RunMode(Enum): """ diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 985ff37de..6268b8a43 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -3,21 +3,22 @@ IStrategy interface This module defines the interface to apply for strategies """ import logging +import warnings from abc import ABC, abstractmethod from datetime import datetime, timezone from enum import Enum from typing import Dict, List, NamedTuple, Optional, Tuple -import warnings import arrow from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider +from freqtrade.exceptions import StrategyError from freqtrade.exchange import timeframe_to_minutes from freqtrade.persistence import Trade +from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets - logger = logging.getLogger(__name__) @@ -59,7 +60,7 @@ class IStrategy(ABC): Attributes you can use: minimal_roi -> Dict: Minimal ROI designed for the strategy stoploss -> float: optimal stoploss designed for the strategy - ticker_interval -> str: value of the ticker interval to use for the strategy + ticker_interval -> str: value of the timeframe (ticker interval) to use with the strategy """ # Strategy interface version # Default to version 2 @@ -112,6 +113,9 @@ class IStrategy(ABC): dp: Optional[DataProvider] = None wallets: Optional[Wallets] = None + # Definition of plot_config. See plotting documentation for more details. + plot_config: Dict = {} + def __init__(self, config: dict) -> None: self.config = config # Dict to determine if analysis is necessary @@ -122,7 +126,7 @@ class IStrategy(ABC): def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param dataframe: DataFrame with data from the exchange :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ @@ -145,6 +149,42 @@ class IStrategy(ABC): :return: DataFrame with sell column """ + def check_buy_timeout(self, pair: str, trade: Trade, order: dict, **kwargs) -> bool: + """ + Check buy timeout function callback. + This method can be used to override the buy-timeout. + It is called whenever a limit buy order has been created, + and is not yet fully filled. + Configuration options in `unfilledtimeout` will be verified before this, + so ensure to set these timeouts high enough. + + When not implemented by a strategy, this simply returns False. + :param pair: Pair the trade is for + :param trade: trade object. + :param order: Order dictionary as returned from CCXT. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the buy-order is cancelled. + """ + return False + + def check_sell_timeout(self, pair: str, trade: Trade, order: dict, **kwargs) -> bool: + """ + Check sell timeout function callback. + This method can be used to override the sell-timeout. + It is called whenever a limit sell order has been created, + and is not yet fully filled. + Configuration options in `unfilledtimeout` will be verified before this, + so ensure to set these timeouts high enough. + + When not implemented by a strategy, this simply returns False. + :param pair: Pair the trade is for + :param trade: trade object. + :param order: Order dictionary as returned from CCXT. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the sell-order is cancelled. + """ + return False + def informative_pairs(self) -> List[Tuple[str, str]]: """ Define additional, informative pair/interval combinations to be cached from the exchange. @@ -168,11 +208,24 @@ class IStrategy(ABC): """ Locks pair until a given timestamp happens. Locked pairs are not analyzed, and are prevented from opening new trades. + Locks can only count up (allowing users to lock pairs for a longer period of time). + To remove a lock from a pair, use `unlock_pair()` :param pair: Pair to lock :param until: datetime in UTC until the pair should be blocked from opening new trades. Needs to be timezone aware `datetime.now(timezone.utc)` """ - self._pair_locked_until[pair] = until + if pair not in self._pair_locked_until or self._pair_locked_until[pair] < until: + self._pair_locked_until[pair] = until + + def unlock_pair(self, pair: str) -> None: + """ + Unlocks a pair previously locked using lock_pair. + Not used by freqtrade itself, but intended to be used if users lock pairs + manually from within the strategy, to allow an easy way to unlock pairs. + :param pair: Unlock pair to allow trading again + """ + if pair in self._pair_locked_until: + del self._pair_locked_until[pair] def is_pair_locked(self, pair: str) -> bool: """ @@ -184,11 +237,11 @@ class IStrategy(ABC): def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Parses the given ticker history and returns a populated DataFrame + Parses the given candle (OHLCV) data and returns a populated DataFrame add several TA indicators and buy signal to it - :param dataframe: Dataframe containing ticker data + :param dataframe: Dataframe containing data from exchange :param metadata: Metadata dictionary with additional data (e.g. 'pair') - :return: DataFrame with ticker data and indicator data + :return: DataFrame of candle (OHLCV) data with indicator data and signals added """ logger.debug("TA Analysis Launched") dataframe = self.advise_indicators(dataframe, metadata) @@ -198,12 +251,12 @@ class IStrategy(ABC): def _analyze_ticker_internal(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Parses the given ticker history and returns a populated DataFrame + Parses the given candle (OHLCV) data and returns a populated DataFrame add several TA indicators and buy signal to it WARNING: Used internally only, may skip analysis if `process_only_new_candles` is set. - :param dataframe: Dataframe containing ticker data + :param dataframe: Dataframe containing data from exchange :param metadata: Metadata dictionary with additional data (e.g. 'pair') - :return: DataFrame with ticker data and indicator data + :return: DataFrame of candle (OHLCV) data with indicator data and signals added """ pair = str(metadata.get('pair')) @@ -225,8 +278,25 @@ class IStrategy(ABC): return dataframe - def get_signal(self, pair: str, interval: str, - dataframe: DataFrame) -> Tuple[bool, bool]: + @staticmethod + def preserve_df(dataframe: DataFrame) -> Tuple[int, float, datetime]: + """ keep some data for dataframes """ + return len(dataframe), dataframe["close"].iloc[-1], dataframe["date"].iloc[-1] + + @staticmethod + def assert_df(dataframe: DataFrame, df_len: int, df_close: float, df_date: datetime): + """ make sure data is unmodified """ + message = "" + if df_len != len(dataframe): + message = "length" + elif df_close != dataframe["close"].iloc[-1]: + message = "last close price" + elif df_date != dataframe["date"].iloc[-1]: + message = "last date" + if message: + raise StrategyError(f"Dataframe returned from strategy has mismatching {message}.") + + def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]: """ Calculates current signal based several technical analysis indicators :param pair: pair in format ANT/BTC @@ -235,31 +305,26 @@ class IStrategy(ABC): :return: (Buy, Sell) A bool-tuple indicating buy/sell signal """ if not isinstance(dataframe, DataFrame) or dataframe.empty: - logger.warning('Empty ticker history for pair %s', pair) + logger.warning('Empty candle (OHLCV) data for pair %s', pair) return False, False + latest_date = dataframe['date'].max() try: - dataframe = self._analyze_ticker_internal(dataframe, {'pair': pair}) - except ValueError as error: - logger.warning( - 'Unable to analyze ticker for pair %s: %s', - pair, - str(error) - ) - return False, False - except Exception as error: - logger.exception( - 'Unexpected error when analyzing ticker for pair %s: %s', - pair, - str(error) - ) + df_len, df_close, df_date = self.preserve_df(dataframe) + dataframe = strategy_safe_wrapper( + self._analyze_ticker_internal, message="" + )(dataframe, {'pair': pair}) + self.assert_df(dataframe, df_len, df_close, df_date) + except StrategyError as error: + logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}") + return False, False if dataframe.empty: logger.warning('Empty dataframe for pair %s', pair) return False, False - latest = dataframe.iloc[-1] + latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1] # Check if dataframe is out of date signal_date = arrow.get(latest['date']) @@ -348,7 +413,7 @@ class IStrategy(ABC): """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not - :param current_profit: current profit in percent + :param current_profit: current profit as ratio """ stop_loss_value = force_stoploss if force_stoploss else self.stoploss @@ -373,9 +438,11 @@ class IStrategy(ABC): trade.adjust_stop_loss(high or current_rate, stop_loss_value) # evaluate if the stoploss was hit if stoploss is not on exchange + # in Dry-Run, this handles stoploss logic as well, as the logic will not be different to + # regular stoploss handling. if ((self.stoploss is not None) and (trade.stop_loss >= current_rate) and - (not self.order_types.get('stoploss_on_exchange'))): + (not self.order_types.get('stoploss_on_exchange') or self.config['dry_run'])): sell_type = SellType.STOP_LOSS @@ -409,8 +476,9 @@ class IStrategy(ABC): def min_roi_reached(self, trade: Trade, current_profit: float, current_time: datetime) -> bool: """ - Based on trade duration, current price and ROI configuration, decides whether bot should - sell. Requires current_profit to be in percent!! + Based on trade duration, current profit of the trade and ROI configuration, + decides whether bot should sell. + :param current_profit: current profit as ratio :return: True if bot should sell at current rate """ # Check if time matches and current rate is above threshold @@ -421,19 +489,22 @@ class IStrategy(ABC): else: return current_profit > roi - def tickerdata_to_dataframe(self, tickerdata: Dict[str, List]) -> Dict[str, DataFrame]: + def ohlcvdata_to_dataframe(self, data: Dict[str, DataFrame]) -> Dict[str, DataFrame]: """ - Creates a dataframe and populates indicators for given ticker data + Creates a dataframe and populates indicators for given candle (OHLCV) data Used by optimize operations only, not during dry / live runs. + Using .copy() to get a fresh copy of the dataframe for every strategy run. + Has positive effects on memory usage for whatever reason - also when + using only one strategy. """ - return {pair: self.advise_indicators(pair_data, {'pair': pair}) - for pair, pair_data in tickerdata.items()} + return {pair: self.advise_indicators(pair_data.copy(), {'pair': pair}) + for pair, pair_data in data.items()} def advise_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Populate indicators that will be used in the Buy and Sell strategy This method should not be overridden. - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param dataframe: Dataframe with data from the exchange :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ diff --git a/freqtrade/strategy/strategy_wrapper.py b/freqtrade/strategy/strategy_wrapper.py new file mode 100644 index 000000000..7b9da9140 --- /dev/null +++ b/freqtrade/strategy/strategy_wrapper.py @@ -0,0 +1,35 @@ +import logging + +from freqtrade.exceptions import StrategyError + +logger = logging.getLogger(__name__) + + +def strategy_safe_wrapper(f, message: str = "", default_retval=None): + """ + Wrapper around user-provided methods and functions. + Caches all exceptions and returns either the default_retval (if it's not None) or raises + a StrategyError exception, which then needs to be handled by the calling method. + """ + def wrapper(*args, **kwargs): + try: + return f(*args, **kwargs) + except ValueError as error: + logger.warning( + f"{message}" + f"Strategy caused the following exception: {error}" + f"{f}" + ) + if default_retval is None: + raise StrategyError(str(error)) from error + return default_retval + except Exception as error: + logger.exception( + f"{message}" + f"Unexpected error {error} calling {f}" + ) + if default_retval is None: + raise StrategyError(str(error)) from error + return default_retval + + return wrapper diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 new file mode 100644 index 000000000..6d3174347 --- /dev/null +++ b/freqtrade/templates/base_config.json.j2 @@ -0,0 +1,61 @@ +{ + "max_open_trades": {{ max_open_trades }}, + "stake_currency": "{{ stake_currency }}", + "stake_amount": {{ stake_amount }}, + "tradable_balance_ratio": 0.99, + "fiat_display_currency": "{{ fiat_display_currency }}", + "ticker_interval": "{{ ticker_interval }}", + "dry_run": {{ dry_run | lower }}, + "cancel_open_orders_on_exit": false, + "unfilledtimeout": { + "buy": 10, + "sell": 30 + }, + "bid_strategy": { + "price_side": "bid", + "ask_last_balance": 0.0, + "use_order_book": false, + "order_book_top": 1, + "check_depth_of_market": { + "enabled": false, + "bids_to_ask_delta": 1 + } + }, + "ask_strategy": { + "price_side": "ask", + "use_order_book": false, + "order_book_min": 1, + "order_book_max": 1, + "use_sell_signal": true, + "sell_profit_only": false, + "ignore_roi_if_buy_signal": false + }, + {{ exchange | indent(4) }}, + "pairlists": [ + {"method": "StaticPairList"} + ], + "edge": { + "enabled": false, + "process_throttle_secs": 3600, + "calculate_since_number_of_days": 7, + "allowed_risk": 0.01, + "stoploss_range_min": -0.01, + "stoploss_range_max": -0.1, + "stoploss_range_step": -0.01, + "minimum_winrate": 0.60, + "minimum_expectancy": 0.20, + "min_trade_number": 10, + "max_trade_duration_minute": 1440, + "remove_pumps": false + }, + "telegram": { + "enabled": {{ telegram | lower }}, + "token": "{{ telegram_token }}", + "chat_id": "{{ telegram_chat_id }}" + }, + "initial_state": "running", + "forcebuy_enable": false, + "internals": { + "process_throttle_secs": 5 + } +} diff --git a/freqtrade/templates/base_hyperopt.py.j2 b/freqtrade/templates/base_hyperopt.py.j2 index 05ba08b81..ec787cbb6 100644 --- a/freqtrade/templates/base_hyperopt.py.j2 +++ b/freqtrade/templates/base_hyperopt.py.j2 @@ -21,7 +21,7 @@ class {{ hyperopt }}(IHyperOpt): """ This is a Hyperopt template to get you started. - More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md + More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ You should: - Add any lib you need to build your hyperopt. @@ -29,11 +29,14 @@ class {{ hyperopt }}(IHyperOpt): You must keep: - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - The roi_space, generate_roi_table, stoploss_space methods are no longer required to be - copied in every custom hyperopt. However, you may override them if you need the - 'roi' and the 'stoploss' spaces that differ from the defaults offered by Freqtrade. - Sample implementation of these methods can be found in - https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py + The methods roi_space, generate_roi_table and stoploss_space are not required + and are provided by default. + However, you may override them if you need 'roi' and 'stoploss' spaces that + differ from the defaults offered by Freqtrade. + Sample implementation of these methods will be copied to `user_data/hyperopts` when + creating the user-data directory using `freqtrade create-userdir --userdir user_data`, + or is available online under the following URL: + https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. """ @staticmethod @@ -63,6 +66,9 @@ class {{ hyperopt }}(IHyperOpt): dataframe['close'], dataframe['sar'] )) + # Check that the candle had volume + conditions.append(dataframe['volume'] > 0) + if conditions: dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -108,6 +114,9 @@ class {{ hyperopt }}(IHyperOpt): dataframe['sar'], dataframe['close'] )) + # Check that the candle had volume + conditions.append(dataframe['volume'] > 0) + if conditions: dataframe.loc[ reduce(lambda x, y: x & y, conditions), diff --git a/freqtrade/templates/base_strategy.py.j2 b/freqtrade/templates/base_strategy.py.j2 index 73a4c7a5a..c37164568 100644 --- a/freqtrade/templates/base_strategy.py.j2 +++ b/freqtrade/templates/base_strategy.py.j2 @@ -47,6 +47,7 @@ class {{ strategy }}(IStrategy): # Trailing stoploss trailing_stop = False + # trailing_only_offset_is_reached = False # trailing_stop_positive = 0.01 # trailing_stop_positive_offset = 0.0 # Disabled / not configured @@ -77,7 +78,7 @@ class {{ strategy }}(IStrategy): 'buy': 'gtc', 'sell': 'gtc' } - + {{ plot_config | indent(4) }} def informative_pairs(self): """ Define additional, informative pair/interval combinations to be cached from the exchange. @@ -98,7 +99,7 @@ class {{ strategy }}(IStrategy): Performance Note: For the best performance be frugal on the number of indicators you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param dataframe: Dataframe with data from the exchange :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ @@ -136,3 +137,4 @@ class {{ strategy }}(IStrategy): ), 'sell'] = 1 return dataframe + {{ additional_methods | indent(4) }} diff --git a/freqtrade/templates/sample_hyperopt.py b/freqtrade/templates/sample_hyperopt.py index f1dcb404a..0b6d030db 100644 --- a/freqtrade/templates/sample_hyperopt.py +++ b/freqtrade/templates/sample_hyperopt.py @@ -20,23 +20,28 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib class SampleHyperOpt(IHyperOpt): """ This is a sample Hyperopt to inspire you. - Feel free to customize it. - More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md + More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ You should: - Rename the class name to some unique name. - Add any methods you want to build your hyperopt. - Add any lib you need to build your hyperopt. + An easier way to get a new hyperopt file is by using + `freqtrade new-hyperopt --hyperopt MyCoolHyperopt`. + You must keep: - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - The roi_space, generate_roi_table, stoploss_space methods are no longer required to be - copied in every custom hyperopt. However, you may override them if you need the - 'roi' and the 'stoploss' spaces that differ from the defaults offered by Freqtrade. - Sample implementation of these methods can be found in - https://github.com/freqtrade/freqtrade/blob/develop/user_data/hyperopts/sample_hyperopt_advanced.py + The methods roi_space, generate_roi_table and stoploss_space are not required + and are provided by default. + However, you may override them if you need 'roi' and 'stoploss' spaces that + differ from the defaults offered by Freqtrade. + Sample implementation of these methods will be copied to `user_data/hyperopts` when + creating the user-data directory using `freqtrade create-userdir --userdir user_data`, + or is available online under the following URL: + https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. """ @staticmethod @@ -73,6 +78,9 @@ class SampleHyperOpt(IHyperOpt): dataframe['close'], dataframe['sar'] )) + # Check that volume is not 0 + conditions.append(dataframe['volume'] > 0) + if conditions: dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -133,6 +141,9 @@ class SampleHyperOpt(IHyperOpt): dataframe['sar'], dataframe['close'] )) + # Check that volume is not 0 + conditions.append(dataframe['volume'] > 0) + if conditions: dataframe.loc[ reduce(lambda x, y: x & y, conditions), diff --git a/freqtrade/templates/sample_hyperopt_advanced.py b/freqtrade/templates/sample_hyperopt_advanced.py index b4bbee3fb..7f05c4430 100644 --- a/freqtrade/templates/sample_hyperopt_advanced.py +++ b/freqtrade/templates/sample_hyperopt_advanced.py @@ -22,7 +22,7 @@ class AdvancedSampleHyperOpt(IHyperOpt): This is a sample hyperopt to inspire you. Feel free to customize it. - More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/hyperopt.md + More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ You should: - Rename the class name to some unique name. @@ -32,8 +32,9 @@ class AdvancedSampleHyperOpt(IHyperOpt): You must keep: - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - The roi_space, generate_roi_table, stoploss_space methods are no longer required to be - copied in every custom hyperopt. However, you may override them if you need the + The methods roi_space, generate_roi_table and stoploss_space are not required + and are provided by default. + However, you may override them if you need the 'roi' and the 'stoploss' spaces that differ from the defaults offered by Freqtrade. This sample illustrates how to override these methods. @@ -92,6 +93,9 @@ class AdvancedSampleHyperOpt(IHyperOpt): dataframe['close'], dataframe['sar'] )) + # Check that volume is not 0 + conditions.append(dataframe['volume'] > 0) + if conditions: dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -152,6 +156,9 @@ class AdvancedSampleHyperOpt(IHyperOpt): dataframe['sar'], dataframe['close'] )) + # Check that volume is not 0 + conditions.append(dataframe['volume'] > 0) + if conditions: dataframe.loc[ reduce(lambda x, y: x & y, conditions), @@ -230,7 +237,7 @@ class AdvancedSampleHyperOpt(IHyperOpt): 'stoploss' optimization hyperspace. """ return [ - Real(-0.5, -0.02, name='stoploss'), + Real(-0.35, -0.02, name='stoploss'), ] @staticmethod @@ -249,8 +256,15 @@ class AdvancedSampleHyperOpt(IHyperOpt): # other 'trailing' hyperspace parameters. Categorical([True], name='trailing_stop'), - Real(0.02, 0.35, name='trailing_stop_positive'), - Real(0.01, 0.1, name='trailing_stop_positive_offset'), + Real(0.01, 0.35, name='trailing_stop_positive'), + + # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive', + # so this intermediate parameter is used as the value of the difference between + # them. The value of the 'trailing_stop_positive_offset' is constructed in the + # generate_trailing_params() method. + # This is similar to the hyperspace dimensions used for constructing the ROI tables. + Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'), + Categorical([True, False], name='trailing_only_offset_is_reached'), ] diff --git a/freqtrade/templates/sample_hyperopt_loss.py b/freqtrade/templates/sample_hyperopt_loss.py index 5a2fb72b6..4173d97f5 100644 --- a/freqtrade/templates/sample_hyperopt_loss.py +++ b/freqtrade/templates/sample_hyperopt_loss.py @@ -27,7 +27,8 @@ class SampleHyperOptLoss(IHyperOptLoss): Defines the default loss function for hyperopt This is intended to give you some inspiration for your own loss function. - The Function needs to return a number (float) - which becomes for better backtest results. + The Function needs to return a number (float) - which becomes smaller for better backtest + results. """ @staticmethod diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index 02bf24e7e..f78489173 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -48,6 +48,7 @@ class SampleStrategy(IStrategy): # Trailing stoploss trailing_stop = False + # trailing_only_offset_is_reached = False # trailing_stop_positive = 0.01 # trailing_stop_positive_offset = 0.0 # Disabled / not configured @@ -79,6 +80,22 @@ class SampleStrategy(IStrategy): 'sell': 'gtc' } + plot_config = { + 'main_plot': { + 'tema': {}, + 'sar': {'color': 'white'}, + }, + 'subplots': { + "MACD": { + 'macd': {'color': 'blue'}, + 'macdsignal': {'color': 'orange'}, + }, + "RSI": { + 'rsi': {'color': 'red'}, + } + } + } + def informative_pairs(self): """ Define additional, informative pair/interval combinations to be cached from the exchange. @@ -99,7 +116,7 @@ class SampleStrategy(IStrategy): Performance Note: For the best performance be frugal on the number of indicators you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param dataframe: Dataframe with data from the exchange :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ @@ -107,24 +124,70 @@ class SampleStrategy(IStrategy): # Momentum Indicators # ------------------------------------ - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - # ADX dataframe['adx'] = ta.ADX(dataframe) + # # Plus Directional Indicator / Movement + # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) + # dataframe['plus_di'] = ta.PLUS_DI(dataframe) + + # # Minus Directional Indicator / Movement + # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) + # dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) # dataframe['aroonup'] = aroon['aroonup'] # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) - # # Awesome oscillator + # # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) - # # Commodity Channel Index: values Oversold:<-100, Overbought:>100 + # # Keltner Channel + # keltner = qtpylib.keltner_channel(dataframe) + # dataframe["kc_upperband"] = keltner["upper"] + # dataframe["kc_lowerband"] = keltner["lower"] + # dataframe["kc_middleband"] = keltner["mid"] + # dataframe["kc_percent"] = ( + # (dataframe["close"] - dataframe["kc_lowerband"]) / + # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) + # ) + # dataframe["kc_width"] = ( + # (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] + # ) + + # # Ultimate Oscillator + # dataframe['uo'] = ta.ULTOSC(dataframe) + + # # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) + # RSI + dataframe['rsi'] = ta.RSI(dataframe) + + # # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) + # rsi = 0.1 * (dataframe['rsi'] - 50) + # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) + + # # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) + # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + + # # Stochastic Slow + # stoch = ta.STOCH(dataframe) + # dataframe['slowd'] = stoch['slowd'] + # dataframe['slowk'] = stoch['slowk'] + + # Stochastic Fast + stoch_fast = ta.STOCHF(dataframe) + dataframe['fastd'] = stoch_fast['fastd'] + dataframe['fastk'] = stoch_fast['fastk'] + + # # Stochastic RSI + # stoch_rsi = ta.STOCHRSI(dataframe) + # dataframe['fastd_rsi'] = stoch_rsi['fastd'] + # dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -134,60 +197,58 @@ class SampleStrategy(IStrategy): # MFI dataframe['mfi'] = ta.MFI(dataframe) - # # Minus Directional Indicator / Movement - # dataframe['minus_dm'] = ta.MINUS_DM(dataframe) - # dataframe['minus_di'] = ta.MINUS_DI(dataframe) - - # # Plus Directional Indicator / Movement - # dataframe['plus_dm'] = ta.PLUS_DM(dataframe) - # dataframe['plus_di'] = ta.PLUS_DI(dataframe) - # dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # # ROC # dataframe['roc'] = ta.ROC(dataframe) - # # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) - # rsi = 0.1 * (dataframe['rsi'] - 50) - # dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) - - # # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) - # dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - - # # Stoch - # stoch = ta.STOCH(dataframe) - # dataframe['slowd'] = stoch['slowd'] - # dataframe['slowk'] = stoch['slowk'] - - # Stoch fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['fastk'] = stoch_fast['fastk'] - - # # Stoch RSI - # stoch_rsi = ta.STOCHRSI(dataframe) - # dataframe['fastd_rsi'] = stoch_rsi['fastd'] - # dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Overlap Studies # ------------------------------------ - # Bollinger bands + # Bollinger Bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] + dataframe["bb_percent"] = ( + (dataframe["close"] - dataframe["bb_lowerband"]) / + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) + ) + dataframe["bb_width"] = ( + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] + ) + + # Bollinger Bands - Weighted (EMA based instead of SMA) + # weighted_bollinger = qtpylib.weighted_bollinger_bands( + # qtpylib.typical_price(dataframe), window=20, stds=2 + # ) + # dataframe["wbb_upperband"] = weighted_bollinger["upper"] + # dataframe["wbb_lowerband"] = weighted_bollinger["lower"] + # dataframe["wbb_middleband"] = weighted_bollinger["mid"] + # dataframe["wbb_percent"] = ( + # (dataframe["close"] - dataframe["wbb_lowerband"]) / + # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) + # ) + # dataframe["wbb_width"] = ( + # (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / + # dataframe["wbb_middleband"] + # ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) + # dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average - # dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) + # dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) + # dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) + # dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) + # dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) + # dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) + # dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) - # SAR Parabol + # Parabolic SAR dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average @@ -247,7 +308,7 @@ class SampleStrategy(IStrategy): # # Chart type # # ------------------------------------ - # # Heikinashi stategy + # # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 2876ea938..dffa308ce 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -6,7 +6,8 @@ "source": [ "# Strategy analysis example\n", "\n", - "Debugging a strategy can be time-consuming. FreqTrade offers helper functions to visualize raw data." + "Debugging a strategy can be time-consuming. Freqtrade offers helper functions to visualize raw data.\n", + "The following assumes you work with SampleStrategy, data for 5m timeframe from Binance and have downloaded them into the data directory in the default location." ] }, { @@ -23,18 +24,21 @@ "outputs": [], "source": [ "from pathlib import Path\n", + "from freqtrade.configuration import Configuration\n", + "\n", "# Customize these according to your needs.\n", "\n", + "# Initialize empty configuration object\n", + "config = Configuration.from_files([])\n", + "# Optionally, use existing configuration file\n", + "# config = Configuration.from_files([\"config.json\"])\n", + "\n", "# Define some constants\n", - "timeframe = \"5m\"\n", + "config[\"ticker_interval\"] = \"5m\"\n", "# Name of the strategy class\n", - "strategy_name = 'SampleStrategy'\n", - "# Path to user data\n", - "user_data_dir = Path('user_data')\n", - "# Location of the strategy\n", - "strategy_location = user_data_dir / 'strategies'\n", + "config[\"strategy\"] = \"SampleStrategy\"\n", "# Location of the data\n", - "data_location = Path(user_data_dir, 'data', 'binance')\n", + "data_location = Path(config['user_data_dir'], 'data', 'binance')\n", "# Pair to analyze - Only use one pair here\n", "pair = \"BTC_USDT\"" ] @@ -49,7 +53,7 @@ "from freqtrade.data.history import load_pair_history\n", "\n", "candles = load_pair_history(datadir=data_location,\n", - " timeframe=timeframe,\n", + " timeframe=config[\"ticker_interval\"],\n", " pair=pair)\n", "\n", "# Confirm success\n", @@ -73,9 +77,7 @@ "source": [ "# Load strategy using values set above\n", "from freqtrade.resolvers import StrategyResolver\n", - "strategy = StrategyResolver({'strategy': strategy_name,\n", - " 'user_data_dir': user_data_dir,\n", - " 'strategy_path': strategy_location}).strategy\n", + "strategy = StrategyResolver.load_strategy(config)\n", "\n", "# Generate buy/sell signals using strategy\n", "df = strategy.analyze_ticker(candles, {'pair': pair})\n", @@ -137,7 +139,7 @@ "from freqtrade.data.btanalysis import load_backtest_data\n", "\n", "# Load backtest results\n", - "trades = load_backtest_data(user_data_dir / \"backtest_results/backtest-result.json\")\n", + "trades = load_backtest_data(config[\"user_data_dir\"] / \"backtest_results/backtest-result.json\")\n", "\n", "# Show value-counts per pair\n", "trades.groupby(\"pair\")[\"sell_reason\"].value_counts()" @@ -188,7 +190,6 @@ "# Analyze the above\n", "parallel_trades = analyze_trade_parallelism(trades, '5m')\n", "\n", - "\n", "parallel_trades.plot()" ] }, @@ -210,11 +211,14 @@ "from freqtrade.plot.plotting import generate_candlestick_graph\n", "# Limit graph period to keep plotly quick and reactive\n", "\n", + "# Filter trades to one pair\n", + "trades_red = trades.loc[trades['pair'] == pair]\n", + "\n", "data_red = data['2019-06-01':'2019-06-10']\n", "# Generate candlestick graph\n", "graph = generate_candlestick_graph(pair=pair,\n", " data=data_red,\n", - " trades=trades,\n", + " trades=trades_red,\n", " indicators1=['sma20', 'ema50', 'ema55'],\n", " indicators2=['rsi', 'macd', 'macdsignal', 'macdhist']\n", " )\n", diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 new file mode 100644 index 000000000..03aa0560c --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -0,0 +1,41 @@ +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 200 + }, + "pair_whitelist": [ + "ALGO/BTC", + "ATOM/BTC", + "BAT/BTC", + "BCH/BTC", + "BRD/BTC", + "EOS/BTC", + "ETH/BTC", + "IOTA/BTC", + "LINK/BTC", + "LTC/BTC", + "NEO/BTC", + "NXS/BTC", + "XMR/BTC", + "XRP/BTC", + "XTZ/BTC" + ], + "pair_blacklist": [ + "BNB/BTC", + "BNB/BUSD", + "BNB/ETH", + "BNB/EUR", + "BNB/NGN", + "BNB/PAX", + "BNB/RUB", + "BNB/TRY", + "BNB/TUSD", + "BNB/USDC", + "BNB/USDS", + "BNB/USDT", + ] +} diff --git a/freqtrade/templates/subtemplates/exchange_bittrex.j2 b/freqtrade/templates/subtemplates/exchange_bittrex.j2 new file mode 100644 index 000000000..7b27318ca --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_bittrex.j2 @@ -0,0 +1,31 @@ +"order_types": { + "buy": "limit", + "sell": "limit", + "emergencysell": "limit", + "stoploss": "limit", + "stoploss_on_exchange": false +}, +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 500 + }, + "pair_whitelist": [ + "ETH/BTC", + "LTC/BTC", + "ETC/BTC", + "DASH/BTC", + "ZEC/BTC", + "XLM/BTC", + "XRP/BTC", + "TRX/BTC", + "ADA/BTC", + "XMR/BTC" + ], + "pair_blacklist": [ + ] +} diff --git a/freqtrade/templates/subtemplates/exchange_generic.j2 b/freqtrade/templates/subtemplates/exchange_generic.j2 new file mode 100644 index 000000000..ade9c2f28 --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_generic.j2 @@ -0,0 +1,15 @@ +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true + }, + "pair_whitelist": [ + + ], + "pair_blacklist": [ + + ] +} diff --git a/freqtrade/templates/subtemplates/exchange_kraken.j2 b/freqtrade/templates/subtemplates/exchange_kraken.j2 new file mode 100644 index 000000000..7139a0830 --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_kraken.j2 @@ -0,0 +1,36 @@ +"download_trades": true, +"exchange": { + "name": "kraken", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 1000 + }, + "pair_whitelist": [ + "ADA/EUR", + "ATOM/EUR", + "BAT/EUR", + "BCH/EUR", + "BTC/EUR", + "DAI/EUR", + "DASH/EUR", + "EOS/EUR", + "ETC/EUR", + "ETH/EUR", + "LINK/EUR", + "LTC/EUR", + "QTUM/EUR", + "REP/EUR", + "WAVES/EUR", + "XLM/EUR", + "XMR/EUR", + "XRP/EUR", + "XTZ/EUR", + "ZEC/EUR" + ], + "pair_blacklist": [ + + ] +} diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 879a2daa0..60a358bec 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -2,24 +2,70 @@ # Momentum Indicators # ------------------------------------ -# RSI -dataframe['rsi'] = ta.RSI(dataframe) - # ADX dataframe['adx'] = ta.ADX(dataframe) +# # Plus Directional Indicator / Movement +# dataframe['plus_dm'] = ta.PLUS_DM(dataframe) +# dataframe['plus_di'] = ta.PLUS_DI(dataframe) + +# # Minus Directional Indicator / Movement +# dataframe['minus_dm'] = ta.MINUS_DM(dataframe) +# dataframe['minus_di'] = ta.MINUS_DI(dataframe) + # # Aroon, Aroon Oscillator # aroon = ta.AROON(dataframe) # dataframe['aroonup'] = aroon['aroonup'] # dataframe['aroondown'] = aroon['aroondown'] # dataframe['aroonosc'] = ta.AROONOSC(dataframe) -# # Awesome oscillator +# # Awesome Oscillator # dataframe['ao'] = qtpylib.awesome_oscillator(dataframe) -# # Commodity Channel Index: values Oversold:<-100, Overbought:>100 +# # Keltner Channel +# keltner = qtpylib.keltner_channel(dataframe) +# dataframe["kc_upperband"] = keltner["upper"] +# dataframe["kc_lowerband"] = keltner["lower"] +# dataframe["kc_middleband"] = keltner["mid"] +# dataframe["kc_percent"] = ( +# (dataframe["close"] - dataframe["kc_lowerband"]) / +# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) +# ) +# dataframe["kc_width"] = ( +# (dataframe["kc_upperband"] - dataframe["kc_lowerband"]) / dataframe["kc_middleband"] +# ) + +# # Ultimate Oscillator +# dataframe['uo'] = ta.ULTOSC(dataframe) + +# # Commodity Channel Index: values [Oversold:-100, Overbought:100] # dataframe['cci'] = ta.CCI(dataframe) +# RSI +dataframe['rsi'] = ta.RSI(dataframe) + +# # Inverse Fisher transform on RSI: values [-1.0, 1.0] (https://goo.gl/2JGGoy) +# rsi = 0.1 * (dataframe['rsi'] - 50) +# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) + +# # Inverse Fisher transform on RSI normalized: values [0.0, 100.0] (https://goo.gl/2JGGoy) +# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) + +# # Stochastic Slow +# stoch = ta.STOCH(dataframe) +# dataframe['slowd'] = stoch['slowd'] +# dataframe['slowk'] = stoch['slowk'] + +# Stochastic Fast +stoch_fast = ta.STOCHF(dataframe) +dataframe['fastd'] = stoch_fast['fastd'] +dataframe['fastk'] = stoch_fast['fastk'] + +# # Stochastic RSI +# stoch_rsi = ta.STOCHRSI(dataframe) +# dataframe['fastd_rsi'] = stoch_rsi['fastd'] +# dataframe['fastk_rsi'] = stoch_rsi['fastk'] + # MACD macd = ta.MACD(dataframe) dataframe['macd'] = macd['macd'] @@ -29,60 +75,57 @@ dataframe['macdhist'] = macd['macdhist'] # MFI dataframe['mfi'] = ta.MFI(dataframe) -# # Minus Directional Indicator / Movement -# dataframe['minus_dm'] = ta.MINUS_DM(dataframe) -# dataframe['minus_di'] = ta.MINUS_DI(dataframe) - -# # Plus Directional Indicator / Movement -# dataframe['plus_dm'] = ta.PLUS_DM(dataframe) -# dataframe['plus_di'] = ta.PLUS_DI(dataframe) -# dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # # ROC # dataframe['roc'] = ta.ROC(dataframe) -# # Inverse Fisher transform on RSI, values [-1.0, 1.0] (https://goo.gl/2JGGoy) -# rsi = 0.1 * (dataframe['rsi'] - 50) -# dataframe['fisher_rsi'] = (np.exp(2 * rsi) - 1) / (np.exp(2 * rsi) + 1) - -# # Inverse Fisher transform on RSI normalized, value [0.0, 100.0] (https://goo.gl/2JGGoy) -# dataframe['fisher_rsi_norma'] = 50 * (dataframe['fisher_rsi'] + 1) - -# # Stoch -# stoch = ta.STOCH(dataframe) -# dataframe['slowd'] = stoch['slowd'] -# dataframe['slowk'] = stoch['slowk'] - -# Stoch fast -stoch_fast = ta.STOCHF(dataframe) -dataframe['fastd'] = stoch_fast['fastd'] -dataframe['fastk'] = stoch_fast['fastk'] - -# # Stoch RSI -# stoch_rsi = ta.STOCHRSI(dataframe) -# dataframe['fastd_rsi'] = stoch_rsi['fastd'] -# dataframe['fastk_rsi'] = stoch_rsi['fastk'] - # Overlap Studies # ------------------------------------ -# Bollinger bands +# Bollinger Bands bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) dataframe['bb_lowerband'] = bollinger['lower'] dataframe['bb_middleband'] = bollinger['mid'] dataframe['bb_upperband'] = bollinger['upper'] +dataframe["bb_percent"] = ( + (dataframe["close"] - dataframe["bb_lowerband"]) / + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) +) +dataframe["bb_width"] = ( + (dataframe["bb_upperband"] - dataframe["bb_lowerband"]) / dataframe["bb_middleband"] +) + +# Bollinger Bands - Weighted (EMA based instead of SMA) +# weighted_bollinger = qtpylib.weighted_bollinger_bands( +# qtpylib.typical_price(dataframe), window=20, stds=2 +# ) +# dataframe["wbb_upperband"] = weighted_bollinger["upper"] +# dataframe["wbb_lowerband"] = weighted_bollinger["lower"] +# dataframe["wbb_middleband"] = weighted_bollinger["mid"] +# dataframe["wbb_percent"] = ( +# (dataframe["close"] - dataframe["wbb_lowerband"]) / +# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) +# ) +# dataframe["wbb_width"] = ( +# (dataframe["wbb_upperband"] - dataframe["wbb_lowerband"]) / dataframe["wbb_middleband"] +# ) # # EMA - Exponential Moving Average # dataframe['ema3'] = ta.EMA(dataframe, timeperiod=3) # dataframe['ema5'] = ta.EMA(dataframe, timeperiod=5) # dataframe['ema10'] = ta.EMA(dataframe, timeperiod=10) +# dataframe['ema21'] = ta.EMA(dataframe, timeperiod=21) # dataframe['ema50'] = ta.EMA(dataframe, timeperiod=50) # dataframe['ema100'] = ta.EMA(dataframe, timeperiod=100) # # SMA - Simple Moving Average -# dataframe['sma'] = ta.SMA(dataframe, timeperiod=40) +# dataframe['sma3'] = ta.SMA(dataframe, timeperiod=3) +# dataframe['sma5'] = ta.SMA(dataframe, timeperiod=5) +# dataframe['sma10'] = ta.SMA(dataframe, timeperiod=10) +# dataframe['sma21'] = ta.SMA(dataframe, timeperiod=21) +# dataframe['sma50'] = ta.SMA(dataframe, timeperiod=50) +# dataframe['sma100'] = ta.SMA(dataframe, timeperiod=100) -# SAR Parabol +# Parabolic SAR dataframe['sar'] = ta.SAR(dataframe) # TEMA - Triple Exponential Moving Average @@ -142,7 +185,7 @@ dataframe['htleadsine'] = hilbert['leadsine'] # # Chart type # # ------------------------------------ -# # Heikinashi stategy +# # Heikin Ashi Strategy # heikinashi = qtpylib.heikinashi(dataframe) # dataframe['ha_open'] = heikinashi['open'] # dataframe['ha_close'] = heikinashi['close'] diff --git a/freqtrade/templates/subtemplates/plot_config_full.j2 b/freqtrade/templates/subtemplates/plot_config_full.j2 new file mode 100644 index 000000000..ab02c7892 --- /dev/null +++ b/freqtrade/templates/subtemplates/plot_config_full.j2 @@ -0,0 +1,18 @@ + +plot_config = { + # Main plot indicators (Moving averages, ...) + 'main_plot': { + 'tema': {}, + 'sar': {'color': 'white'}, + }, + 'subplots': { + # Subplots - each dict defines one additional plot + "MACD": { + 'macd': {'color': 'blue'}, + 'macdsignal': {'color': 'orange'}, + }, + "RSI": { + 'rsi': {'color': 'red'}, + } + } +} diff --git a/user_data/backtest_data/.gitkeep b/freqtrade/templates/subtemplates/plot_config_minimal.j2 similarity index 100% rename from user_data/backtest_data/.gitkeep rename to freqtrade/templates/subtemplates/plot_config_minimal.j2 diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 new file mode 100644 index 000000000..0ca35e117 --- /dev/null +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -0,0 +1,40 @@ + +def check_buy_timeout(self, pair: str, trade: 'Trade', order: dict, **kwargs) -> bool: + """ + Check buy timeout function callback. + This method can be used to override the buy-timeout. + It is called whenever a limit buy order has been created, + and is not yet fully filled. + Configuration options in `unfilledtimeout` will be verified before this, + so ensure to set these timeouts high enough. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, this simply returns False. + :param pair: Pair the trade is for + :param trade: trade object. + :param order: Order dictionary as returned from CCXT. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the buy-order is cancelled. + """ + return False + +def check_sell_timeout(self, pair: str, trade: 'Trade', order: dict, **kwargs) -> bool: + """ + Check sell timeout function callback. + This method can be used to override the sell-timeout. + It is called whenever a limit sell order has been created, + and is not yet fully filled. + Configuration options in `unfilledtimeout` will be verified before this, + so ensure to set these timeouts high enough. + + For full documentation please go to https://www.freqtrade.io/en/latest/strategy-advanced/ + + When not implemented by a strategy, this simply returns False. + :param pair: Pair the trade is for + :param trade: trade object. + :param order: Order dictionary as returned from CCXT. + :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. + :return bool: When True is returned, then the sell-order is cancelled. + """ + return False diff --git a/freqtrade/templates/subtemplates/strategy_methods_empty.j2 b/freqtrade/templates/subtemplates/strategy_methods_empty.j2 new file mode 100644 index 000000000..e69de29bb diff --git a/freqtrade/utils.py b/freqtrade/utils.py deleted file mode 100644 index 9e01c7ea6..000000000 --- a/freqtrade/utils.py +++ /dev/null @@ -1,461 +0,0 @@ -import csv -import logging -import sys -from collections import OrderedDict -from operator import itemgetter -from pathlib import Path -from typing import Any, Dict, List - -import arrow -import rapidjson -from colorama import init as colorama_init -from tabulate import tabulate - -from freqtrade import OperationalException -from freqtrade.configuration import (Configuration, TimeRange, - remove_credentials) -from freqtrade.configuration.directory_operations import (copy_sample_files, - create_userdata_dir) -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGY -from freqtrade.data.history import (convert_trades_to_ohlcv, - refresh_backtest_ohlcv_data, - refresh_backtest_trades_data) -from freqtrade.exchange import (available_exchanges, ccxt_exchanges, - market_is_active, symbol_is_pair) -from freqtrade.misc import plural, render_template -from freqtrade.resolvers import ExchangeResolver -from freqtrade.state import RunMode - -logger = logging.getLogger(__name__) - - -def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str, Any]: - """ - Prepare the configuration for utils subcommands - :param args: Cli args from Arguments() - :return: Configuration - """ - configuration = Configuration(args, method) - config = configuration.get_config() - - # Ensure we do not use Exchange credentials - remove_credentials(config) - - return config - - -def start_trading(args: Dict[str, Any]) -> int: - """ - Main entry point for trading mode - """ - from freqtrade.worker import Worker - # Load and run worker - worker = None - try: - worker = Worker(args) - worker.run() - except KeyboardInterrupt: - logger.info('SIGINT received, aborting ...') - finally: - if worker: - logger.info("worker found ... calling exit") - worker.exit() - return 0 - - -def start_list_exchanges(args: Dict[str, Any]) -> None: - """ - Print available exchanges - :param args: Cli args from Arguments() - :return: None - """ - exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges() - if args['print_one_column']: - print('\n'.join(exchanges)) - else: - if args['list_exchanges_all']: - print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}") - else: - print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}") - - -def start_create_userdir(args: Dict[str, Any]) -> None: - """ - Create "user_data" directory to contain user data strategies, hyperopt, ...) - :param args: Cli args from Arguments() - :return: None - """ - if "user_data_dir" in args and args["user_data_dir"]: - userdir = create_userdata_dir(args["user_data_dir"], create_dir=True) - copy_sample_files(userdir, overwrite=args["reset"]) - else: - logger.warning("`create-userdir` requires --userdir to be set.") - sys.exit(1) - - -def deploy_new_strategy(strategy_name, strategy_path: Path, subtemplate: str): - """ - Deploy new strategy from template to strategy_path - """ - indicators = render_template(templatefile=f"subtemplates/indicators_{subtemplate}.j2",) - buy_trend = render_template(templatefile=f"subtemplates/buy_trend_{subtemplate}.j2",) - sell_trend = render_template(templatefile=f"subtemplates/sell_trend_{subtemplate}.j2",) - - strategy_text = render_template(templatefile='base_strategy.py.j2', - arguments={"strategy": strategy_name, - "indicators": indicators, - "buy_trend": buy_trend, - "sell_trend": sell_trend, - }) - - logger.info(f"Writing strategy to `{strategy_path}`.") - strategy_path.write_text(strategy_text) - - -def start_new_strategy(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if "strategy" in args and args["strategy"]: - if args["strategy"] == "DefaultStrategy": - raise OperationalException("DefaultStrategy is not allowed as name.") - - new_path = config['user_data_dir'] / USERPATH_STRATEGY / (args["strategy"] + ".py") - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Strategy Name.") - - deploy_new_strategy(args['strategy'], new_path, args['template']) - - else: - raise OperationalException("`new-strategy` requires --strategy to be set.") - - -def deploy_new_hyperopt(hyperopt_name, hyperopt_path: Path, subtemplate: str): - """ - Deploys a new hyperopt template to hyperopt_path - """ - buy_guards = render_template( - templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2",) - sell_guards = render_template( - templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2",) - buy_space = render_template( - templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2",) - sell_space = render_template( - templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2",) - - strategy_text = render_template(templatefile='base_hyperopt.py.j2', - arguments={"hyperopt": hyperopt_name, - "buy_guards": buy_guards, - "sell_guards": sell_guards, - "buy_space": buy_space, - "sell_space": sell_space, - }) - - logger.info(f"Writing hyperopt to `{hyperopt_path}`.") - hyperopt_path.write_text(strategy_text) - - -def start_new_hyperopt(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if "hyperopt" in args and args["hyperopt"]: - if args["hyperopt"] == "DefaultHyperopt": - raise OperationalException("DefaultHyperopt is not allowed as name.") - - new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args["hyperopt"] + ".py") - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Strategy Name.") - deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) - else: - raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") - - -def start_download_data(args: Dict[str, Any]) -> None: - """ - Download data (former download_backtest_data.py script) - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - timerange = TimeRange() - if 'days' in config: - time_since = arrow.utcnow().shift(days=-config['days']).strftime("%Y%m%d") - timerange = TimeRange.parse_timerange(f'{time_since}-') - - if 'pairs' not in config: - raise OperationalException( - "Downloading data requires a list of pairs. " - "Please check the documentation on how to configure this.") - - dl_path = Path(config['datadir']) - logger.info(f'About to download pairs: {config["pairs"]}, ' - f'intervals: {config["timeframes"]} to {dl_path}') - - pairs_not_available: List[str] = [] - - # Init exchange - exchange = ExchangeResolver(config['exchange']['name'], config).exchange - try: - - if config.get('download_trades'): - pairs_not_available = refresh_backtest_trades_data( - exchange, pairs=config["pairs"], datadir=Path(config['datadir']), - timerange=timerange, erase=config.get("erase")) - - # Convert downloaded trade data to different timeframes - convert_trades_to_ohlcv( - pairs=config["pairs"], timeframes=config["timeframes"], - datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) - else: - pairs_not_available = refresh_backtest_ohlcv_data( - exchange, pairs=config["pairs"], timeframes=config["timeframes"], - datadir=Path(config['datadir']), timerange=timerange, erase=config.get("erase")) - - except KeyboardInterrupt: - sys.exit("SIGINT received, aborting ...") - - finally: - if pairs_not_available: - logger.info(f"Pairs [{','.join(pairs_not_available)}] not available " - f"on exchange {exchange.name}.") - - -def start_list_timeframes(args: Dict[str, Any]) -> None: - """ - Print ticker intervals (timeframes) available on Exchange - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - # Do not use ticker_interval set in the config - config['ticker_interval'] = None - - # Init exchange - exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange - - if args['print_one_column']: - print('\n'.join(exchange.timeframes)) - else: - print(f"Timeframes available for the exchange `{exchange.name}`: " - f"{', '.join(exchange.timeframes)}") - - -def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: - """ - Print pairs/markets on the exchange - :param args: Cli args from Arguments() - :param pairs_only: if True print only pairs, otherwise print all instruments (markets) - :return: None - """ - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - # Init exchange - exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange - - # By default only active pairs/markets are to be shown - active_only = not args.get('list_pairs_all', False) - - base_currencies = args.get('base_currencies', []) - quote_currencies = args.get('quote_currencies', []) - - try: - pairs = exchange.get_markets(base_currencies=base_currencies, - quote_currencies=quote_currencies, - pairs_only=pairs_only, - active_only=active_only) - # Sort the pairs/markets by symbol - pairs = OrderedDict(sorted(pairs.items())) - except Exception as e: - raise OperationalException(f"Cannot get markets. Reason: {e}") from e - - else: - summary_str = ((f"Exchange {exchange.name} has {len(pairs)} ") + - ("active " if active_only else "") + - (plural(len(pairs), "pair" if pairs_only else "market")) + - (f" with {', '.join(base_currencies)} as base " - f"{plural(len(base_currencies), 'currency', 'currencies')}" - if base_currencies else "") + - (" and" if base_currencies and quote_currencies else "") + - (f" with {', '.join(quote_currencies)} as quote " - f"{plural(len(quote_currencies), 'currency', 'currencies')}" - if quote_currencies else "")) - - headers = ["Id", "Symbol", "Base", "Quote", "Active", - *(['Is pair'] if not pairs_only else [])] - - tabular_data = [] - for _, v in pairs.items(): - tabular_data.append({'Id': v['id'], 'Symbol': v['symbol'], - 'Base': v['base'], 'Quote': v['quote'], - 'Active': market_is_active(v), - **({'Is pair': symbol_is_pair(v['symbol'])} - if not pairs_only else {})}) - - if (args.get('print_one_column', False) or - args.get('list_pairs_print_json', False) or - args.get('print_csv', False)): - # Print summary string in the log in case of machine-readable - # regular formats. - logger.info(f"{summary_str}.") - else: - # Print empty string separating leading logs and output in case of - # human-readable formats. - print() - - if len(pairs): - if args.get('print_list', False): - # print data as a list, with human-readable summary - print(f"{summary_str}: {', '.join(pairs.keys())}.") - elif args.get('print_one_column', False): - print('\n'.join(pairs.keys())) - elif args.get('list_pairs_print_json', False): - print(rapidjson.dumps(list(pairs.keys()), default=str)) - elif args.get('print_csv', False): - writer = csv.DictWriter(sys.stdout, fieldnames=headers) - writer.writeheader() - writer.writerows(tabular_data) - else: - # print data as a table, with the human-readable summary - print(f"{summary_str}:") - print(tabulate(tabular_data, headers='keys', tablefmt='pipe')) - elif not (args.get('print_one_column', False) or - args.get('list_pairs_print_json', False) or - args.get('print_csv', False)): - print(f"{summary_str}.") - - -def start_test_pairlist(args: Dict[str, Any]) -> None: - """ - Test Pairlist configuration - """ - from freqtrade.pairlist.pairlistmanager import PairListManager - config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) - - exchange = ExchangeResolver(config['exchange']['name'], config, validate=False).exchange - - quote_currencies = args.get('quote_currencies') - if not quote_currencies: - quote_currencies = [config.get('stake_currency')] - results = {} - for curr in quote_currencies: - config['stake_currency'] = curr - # Do not use ticker_interval set in the config - pairlists = PairListManager(exchange, config) - pairlists.refresh_pairlist() - results[curr] = pairlists.whitelist - - for curr, pairlist in results.items(): - if not args.get('print_one_column', False): - print(f"Pairs for {curr}: ") - - if args.get('print_one_column', False): - print('\n'.join(pairlist)) - elif args.get('list_pairs_print_json', False): - print(rapidjson.dumps(list(pairlist), default=str)) - else: - print(pairlist) - - -def start_hyperopt_list(args: Dict[str, Any]) -> None: - """ - List hyperopt epochs previously evaluated - """ - from freqtrade.optimize.hyperopt import Hyperopt - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) - print_colorized = config.get('print_colorized', False) - print_json = config.get('print_json', False) - no_details = config.get('hyperopt_list_no_details', False) - no_header = False - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - - # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) - - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) - - # TODO: fetch the interval for epochs to print from the cli option - epoch_start, epoch_stop = 0, None - - if print_colorized: - colorama_init(autoreset=True) - - try: - # Human-friendly indexes used here (starting from 1) - for val in trials[epoch_start:epoch_stop]: - Hyperopt.print_results_explanation(val, total_epochs, not only_best, print_colorized) - - except KeyboardInterrupt: - print('User interrupted..') - - if trials and not no_details: - sorted_trials = sorted(trials, key=itemgetter('loss')) - results = sorted_trials[0] - Hyperopt.print_epoch_details(results, total_epochs, print_json, no_header) - - -def start_hyperopt_show(args: Dict[str, Any]) -> None: - """ - Show details of a hyperopt epoch previously evaluated - """ - from freqtrade.optimize.hyperopt import Hyperopt - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - only_best = config.get('hyperopt_list_best', False) - only_profitable = config.get('hyperopt_list_profitable', False) - no_header = config.get('hyperopt_show_no_header', False) - - trials_file = (config['user_data_dir'] / - 'hyperopt_results' / 'hyperopt_results.pickle') - - # Previous evaluations - trials = Hyperopt.load_previous_results(trials_file) - total_epochs = len(trials) - - trials = _hyperopt_filter_trials(trials, only_best, only_profitable) - trials_epochs = len(trials) - - n = config.get('hyperopt_show_index', -1) - if n > trials_epochs: - raise OperationalException( - f"The index of the epoch to show should be less than {trials_epochs + 1}.") - if n < -trials_epochs: - raise OperationalException( - f"The index of the epoch to show should be greater than {-trials_epochs - 1}.") - - # Translate epoch index from human-readable format to pythonic - if n > 0: - n -= 1 - - print_json = config.get('print_json', False) - - if trials: - val = trials[n] - Hyperopt.print_epoch_details(val, total_epochs, print_json, no_header, - header_str="Epoch details") - - -def _hyperopt_filter_trials(trials: List, only_best: bool, only_profitable: bool) -> List: - """ - Filter our items from the list of hyperopt results - """ - if only_best: - trials = [x for x in trials if x['is_best']] - if only_profitable: - trials = [x for x in trials if x['results_metrics']['profit'] > 0] - - logger.info(f"{len(trials)} " + - ("best " if only_best else "") + - ("profitable " if only_profitable else "") + - "epochs found.") - - return trials diff --git a/freqtrade/vendor/qtpylib/indicators.py b/freqtrade/vendor/qtpylib/indicators.py index b3b2ac533..bef140396 100644 --- a/freqtrade/vendor/qtpylib/indicators.py +++ b/freqtrade/vendor/qtpylib/indicators.py @@ -288,9 +288,9 @@ def rolling_min(series, window=14, min_periods=None): def rolling_max(series, window=14, min_periods=None): min_periods = window if min_periods is None else min_periods try: - return series.rolling(window=window, min_periods=min_periods).min() + return series.rolling(window=window, min_periods=min_periods).max() except Exception as e: # noqa: F841 - return pd.Series(series).rolling(window=window, min_periods=min_periods).min() + return pd.Series(series).rolling(window=window, min_periods=min_periods).max() # --------------------------------------------- diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index dd706438f..b913155bc 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -2,7 +2,10 @@ """ Wallet """ import logging -from typing import Dict, NamedTuple, Any +from typing import Any, Dict, NamedTuple + +import arrow + from freqtrade.exchange import Exchange from freqtrade.persistence import Trade @@ -24,27 +27,24 @@ class Wallets: self._exchange = exchange self._wallets: Dict[str, Wallet] = {} self.start_cap = config['dry_run_wallet'] - + self._last_wallet_refresh = 0 self.update() - def get_free(self, currency) -> float: - + def get_free(self, currency: str) -> float: balance = self._wallets.get(currency) if balance and balance.free: return balance.free else: return 0 - def get_used(self, currency) -> float: - + def get_used(self, currency: str) -> float: balance = self._wallets.get(currency) if balance and balance.used: return balance.used else: return 0 - def get_total(self, currency) -> float: - + def get_total(self, currency: str) -> float: balance = self._wallets.get(currency) if balance and balance.total: return balance.total @@ -58,13 +58,15 @@ class Wallets: - Subtract currently tied up stake_amount in open trades - update balances for currencies currently in trades """ + # Recreate _wallets to reset closed trade balances + _wallets = {} closed_trades = Trade.get_trades(Trade.is_open.is_(False)).all() open_trades = Trade.get_trades(Trade.is_open.is_(True)).all() tot_profit = sum([trade.calc_profit() for trade in closed_trades]) tot_in_trades = sum([trade.stake_amount for trade in open_trades]) current_stake = self.start_cap + tot_profit - tot_in_trades - self._wallets[self._config['stake_currency']] = Wallet( + _wallets[self._config['stake_currency']] = Wallet( self._config['stake_currency'], current_stake, 0, @@ -72,16 +74,16 @@ class Wallets: ) for trade in open_trades: - curr = trade.pair.split('/')[0] - self._wallets[curr] = Wallet( + curr = self._exchange.get_pair_base_currency(trade.pair) + _wallets[curr] = Wallet( curr, trade.amount, 0, trade.amount ) + self._wallets = _wallets def _update_live(self) -> None: - balances = self._exchange.get_balances() for currency in balances: @@ -92,12 +94,21 @@ class Wallets: balances[currency].get('total', None) ) - def update(self) -> None: - if self._config['dry_run']: - self._update_dry() - else: - self._update_live() - logger.info('Wallets synced.') + def update(self, require_update: bool = True) -> None: + """ + Updates wallets from the configured version. + By default, updates from the exchange. + Update-skipping should only be used for user-invoked /balance calls, since + for trading operations, the latest balance is needed. + :param require_update: Allow skipping an update if balances were recently refreshed + """ + if (require_update or (self._last_wallet_refresh + 3600 < arrow.utcnow().timestamp)): + if self._config['dry_run']: + self._update_dry() + else: + self._update_live() + logger.info('Wallets synced.') + self._last_wallet_refresh = arrow.utcnow().timestamp def get_all_balances(self) -> Dict[str, Any]: return self._wallets diff --git a/freqtrade/worker.py b/freqtrade/worker.py index 8e4be9d43..3f5ab734e 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -4,15 +4,15 @@ Main Freqtrade worker class. import logging import time import traceback +from os import getpid from typing import Any, Callable, Dict, Optional import sdnotify -from freqtrade import (OperationalException, TemporaryError, __version__, - constants) +from freqtrade import __version__, constants from freqtrade.configuration import Configuration +from freqtrade.exceptions import OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.rpc import RPCMessageType from freqtrade.state import State logger = logging.getLogger(__name__) @@ -23,20 +23,21 @@ class Worker: Freqtradebot worker class """ - def __init__(self, args: Dict[str, Any], config=None) -> None: + def __init__(self, args: Dict[str, Any], config: Dict[str, Any] = None) -> None: """ Init all variables and objects the bot needs to work """ - logger.info('Starting worker %s', __version__) + logger.info(f"Starting worker {__version__}") self._args = args self._config = config self._init(False) + self.last_throttle_start_time: float = 0 + self._heartbeat_msg: float = 0 + # Tell systemd that we completed initialization phase - if self._sd_notify: - logger.debug("sd_notify: READY=1") - self._sd_notify.notify("READY=1") + self._notify("READY=1") def _init(self, reconfig: bool) -> None: """ @@ -49,21 +50,22 @@ class Worker: # Init the instance of the bot self.freqtrade = FreqtradeBot(self._config) - self._throttle_secs = self._config.get('internals', {}).get( - 'process_throttle_secs', - constants.PROCESS_THROTTLE_SECS - ) + internals_config = self._config.get('internals', {}) + self._throttle_secs = internals_config.get('process_throttle_secs', + constants.PROCESS_THROTTLE_SECS) + self._heartbeat_interval = internals_config.get('heartbeat_interval', 60) self._sd_notify = sdnotify.SystemdNotifier() if \ self._config.get('internals', {}).get('sd_notify', False) else None - @property - def state(self) -> State: - return self.freqtrade.state - - @state.setter - def state(self, value: State) -> None: - self.freqtrade.state = value + def _notify(self, message: str) -> None: + """ + Removes the need to verify in all occurances if sd_notify is enabled + :param message: Message to send to systemd if it's enabled. + """ + if self._sd_notify: + logger.debug(f"sd_notify: {message}") + self._sd_notify.notify(message) def run(self) -> None: state = None @@ -72,62 +74,69 @@ class Worker: if state == State.RELOAD_CONF: self._reconfigure() - def _worker(self, old_state: Optional[State], throttle_secs: Optional[float] = None) -> State: + def _worker(self, old_state: Optional[State]) -> State: """ - Trading routine that must be run at each loop + The main routine that runs each throttling iteration and handles the states. :param old_state: the previous service state from the previous call :return: current service state """ state = self.freqtrade.state - if throttle_secs is None: - throttle_secs = self._throttle_secs # Log state transition if state != old_state: - self.freqtrade.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'{state.name.lower()}' - }) - logger.info('Changing state to: %s', state.name) + self.freqtrade.notify_status(f'{state.name.lower()}') + + logger.info(f"Changing state to: {state.name}") if state == State.RUNNING: self.freqtrade.startup() + # Reset heartbeat timestamp to log the heartbeat message at + # first throttling iteration when the state changes + self._heartbeat_msg = 0 + if state == State.STOPPED: # Ping systemd watchdog before sleeping in the stopped state - if self._sd_notify: - logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: STOPPED.") - self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: STOPPED.") + self._notify("WATCHDOG=1\nSTATUS=State: STOPPED.") - time.sleep(throttle_secs) + self._throttle(func=self._process_stopped, throttle_secs=self._throttle_secs) elif state == State.RUNNING: # Ping systemd watchdog before throttling - if self._sd_notify: - logger.debug("sd_notify: WATCHDOG=1\\nSTATUS=State: RUNNING.") - self._sd_notify.notify("WATCHDOG=1\nSTATUS=State: RUNNING.") + self._notify("WATCHDOG=1\nSTATUS=State: RUNNING.") - self._throttle(func=self._process, min_secs=throttle_secs) + self._throttle(func=self._process_running, throttle_secs=self._throttle_secs) + + if self._heartbeat_interval: + now = time.time() + if (now - self._heartbeat_msg) > self._heartbeat_interval: + logger.info(f"Bot heartbeat. PID={getpid()}, " + f"version='{__version__}', state='{state.name}'") + self._heartbeat_msg = now return state - def _throttle(self, func: Callable[..., Any], min_secs: float, *args, **kwargs) -> Any: + def _throttle(self, func: Callable[..., Any], throttle_secs: float, *args, **kwargs) -> Any: """ Throttles the given callable that it takes at least `min_secs` to finish execution. :param func: Any callable - :param min_secs: minimum execution time in seconds - :return: Any + :param throttle_secs: throttling interation execution time limit in seconds + :return: Any (result of execution of func) """ - start = time.time() + self.last_throttle_start_time = time.time() + logger.debug("========================================") result = func(*args, **kwargs) - end = time.time() - duration = max(min_secs - (end - start), 0.0) - logger.debug('Throttling %s for %.2f seconds', func.__name__, duration) - time.sleep(duration) + time_passed = time.time() - self.last_throttle_start_time + sleep_duration = max(throttle_secs - time_passed, 0.0) + logger.debug(f"Throttling with '{func.__name__}()': sleep for {sleep_duration:.2f} s, " + f"last iteration took {time_passed:.2f} s.") + time.sleep(sleep_duration) return result - def _process(self) -> None: - logger.debug("========================================") + def _process_stopped(self) -> None: + self.freqtrade.process_stopped() + + def _process_running(self) -> None: try: self.freqtrade.process() except TemporaryError as error: @@ -136,10 +145,9 @@ class Worker: except OperationalException: tb = traceback.format_exc() hint = 'Issue `/start` if you think it is safe to restart.' - self.freqtrade.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': f'OperationalException:\n```\n{tb}```{hint}' - }) + + self.freqtrade.notify_status(f'OperationalException:\n```\n{tb}```{hint}') + logger.exception('OperationalException. Stopping trader ...') self.freqtrade.state = State.STOPPED @@ -149,9 +157,7 @@ class Worker: replaces it with the new instance """ # Tell systemd that we initiated reconfiguration - if self._sd_notify: - logger.debug("sd_notify: RELOADING=1") - self._sd_notify.notify("RELOADING=1") + self._notify("RELOADING=1") # Clean up current freqtrade modules self.freqtrade.cleanup() @@ -159,25 +165,15 @@ class Worker: # Load and validate config and create new instance of the bot self._init(True) - self.freqtrade.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': 'config reloaded' - }) + self.freqtrade.notify_status('config reloaded') # Tell systemd that we completed reconfiguration - if self._sd_notify: - logger.debug("sd_notify: READY=1") - self._sd_notify.notify("READY=1") + self._notify("READY=1") def exit(self) -> None: # Tell systemd that we are exiting now - if self._sd_notify: - logger.debug("sd_notify: STOPPING=1") - self._sd_notify.notify("STOPPING=1") + self._notify("STOPPING=1") if self.freqtrade: - self.freqtrade.rpc.send_msg({ - 'type': RPCMessageType.STATUS_NOTIFICATION, - 'status': 'process died' - }) + self.freqtrade.notify_status('process died') self.freqtrade.cleanup() diff --git a/mkdocs.yml b/mkdocs.yml index d53687c64..ae24e150c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,8 +1,8 @@ site_name: Freqtrade nav: - - About: index.md - - Installation: installation.md + - Home: index.md - Installation Docker: docker.md + - Installation: installation.md - Configuration: configuration.md - Strategy Customization: strategy-customization.md - Stoploss: stoploss.md @@ -24,6 +24,7 @@ nav: - Plotting: plotting.md - SQL Cheatsheet: sql_cheatsheet.md - Advanced Post-installation Tasks: advanced-setup.md + - Advanced Strategy: strategy-advanced.md - Advanced Hyperopt: advanced-hyperopt.md - Sandbox Testing: sandbox-testing.md - Deprecated Features: deprecated.md diff --git a/requirements-common.txt b/requirements-common.txt index a6d9a6f5e..a2038d95e 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -1,18 +1,18 @@ # requirements without requirements installable via conda # mainly used for Raspberry pi installs -ccxt==1.20.84 -SQLAlchemy==1.3.11 -python-telegram-bot==12.2.0 -arrow==0.15.4 -cachetools==4.0.0 -requests==2.22.0 -urllib3==1.25.7 -wrapt==1.11.2 +ccxt==1.27.91 +SQLAlchemy==1.3.17 +python-telegram-bot==12.7 +arrow==0.15.6 +cachetools==4.1.0 +requests==2.23.0 +urllib3==1.25.9 +wrapt==1.12.1 jsonschema==3.2.0 -TA-Lib==0.4.17 -tabulate==0.8.6 -coinmarketcap==5.0.3 -jinja2==2.10.3 +TA-Lib==0.4.18 +tabulate==0.8.7 +pycoingecko==1.2.0 +jinja2==2.11.2 # find first, C search in arrays py_find_1st==1.1.4 @@ -24,7 +24,12 @@ python-rapidjson==0.9.1 sdnotify==0.3.2 # Api server -flask==1.1.1 +flask==1.1.2 +flask-jwt-extended==3.24.1 +flask-cors==3.0.8 # Support for colorized terminal output colorama==0.4.3 +# Building config files interactively +questionary==1.5.2 +prompt-toolkit==3.0.5 diff --git a/requirements-dev.txt b/requirements-dev.txt index fe5b4e369..a37ac42ae 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,15 +3,15 @@ -r requirements-plot.txt -r requirements-hyperopt.txt -coveralls==1.9.2 -flake8==3.7.9 +coveralls==2.0.0 +flake8==3.8.1 flake8-type-annotations==0.1.0 -flake8-tidy-imports==3.1.0 -mypy==0.750 -pytest==5.3.2 -pytest-asyncio==0.10.0 +flake8-tidy-imports==4.1.0 +mypy==0.770 +pytest==5.4.2 +pytest-asyncio==0.12.0 pytest-cov==2.8.1 -pytest-mock==1.13.0 +pytest-mock==3.1.0 pytest-random-order==1.0.4 # Convert jupyter notebooks to markdown documents diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index b2428e37d..c3dda6c4b 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -2,8 +2,9 @@ -r requirements.txt # Required for hyperopt -scipy==1.3.3 -scikit-learn==0.22 -scikit-optimize==0.5.2 +scipy==1.4.1 +scikit-learn==0.22.2.post1 +scikit-optimize==0.7.4 filelock==3.0.12 -joblib==0.14.1 +joblib==0.15.1 +progressbar2==3.51.3 diff --git a/requirements-plot.txt b/requirements-plot.txt index 415d4b888..d81239053 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.4.1 +plotly==4.7.1 diff --git a/requirements.txt b/requirements.txt index ebf27abd4..18cab206b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Load common requirements -r requirements-common.txt -numpy==1.17.4 -pandas==0.25.3 +numpy==1.18.4 +pandas==1.0.3 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ccb33604f..b26c32479 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -156,6 +156,14 @@ class FtRestClient(): """ return self._get("show_config") + def trades(self, limit=None): + """Return trades history. + + :param limit: Limits trades to the X last trades. No limit to get all the trades. + :return: json object + """ + return self._get("trades", params={"limit": limit} if limit else 0) + def whitelist(self): """Show the current whitelist. diff --git a/setup.py b/setup.py index 3710bcdc0..20963a15f 100644 --- a/setup.py +++ b/setup.py @@ -16,14 +16,15 @@ if readme_file.is_file(): readme_long = (Path(__file__).parent / "README.md").read_text() # Requirements used for submodules -api = ['flask'] +api = ['flask', 'flask-jwt-extended', 'flask-cors'] plot = ['plotly>=4.0'] hyperopt = [ 'scipy', 'scikit-learn', - 'scikit-optimize', + 'scikit-optimize>=0.7.0', 'filelock', 'joblib', + 'progressbar2', ] develop = [ @@ -59,7 +60,7 @@ setup(name='freqtrade', license='GPLv3', packages=['freqtrade'], setup_requires=['pytest-runner', 'numpy'], - tests_require=['pytest', 'pytest-mock', 'pytest-cov'], + tests_require=['pytest', 'pytest-asyncio', 'pytest-cov', 'pytest-mock', ], install_requires=[ # from requirements-common.txt 'ccxt>=1.18.1080', @@ -73,12 +74,14 @@ setup(name='freqtrade', 'jsonschema', 'TA-Lib', 'tabulate', - 'coinmarketcap', + 'pycoingecko', 'py_find_1st', 'python-rapidjson', 'sdnotify', 'colorama', 'jinja2', + 'questionary', + 'prompt-toolkit', # from requirements.txt 'numpy', 'pandas', @@ -99,8 +102,12 @@ setup(name='freqtrade', ], }, classifiers=[ - 'Programming Language :: Python :: 3.6', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Topic :: Office/Business :: Financial :: Investment', + 'Environment :: Console', 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Operating System :: MacOS', + 'Operating System :: Unix', + 'Topic :: Office/Business :: Financial :: Investment', ]) diff --git a/setup.sh b/setup.sh index c4b6e074a..918c41e6b 100755 --- a/setup.sh +++ b/setup.sh @@ -17,6 +17,14 @@ function check_installed_python() { exit 2 fi + which python3.8 + if [ $? -eq 0 ]; then + echo "using Python 3.8" + PYTHON=python3.8 + check_installed_pip + return + fi + which python3.7 if [ $? -eq 0 ]; then echo "using Python 3.7" @@ -215,27 +223,8 @@ function config_generator() { function config() { echo "-------------------------" - echo "Generating config file" + echo "Please use 'freqtrade new-config -c config.json' to generate a new configuration file." echo "-------------------------" - if [ -f config.json ] - then - read -p "A config file already exist, do you want to override it [y/N]? " - if [[ $REPLY =~ ^[Yy]$ ]] - then - config_generator - else - echo "Configuration of config.json ignored." - fi - else - config_generator - fi - - echo - echo "-------------------------" - echo "Config file generated" - echo "-------------------------" - echo "Edit ./config.json to modify Pair and other configurations." - echo } function install() { @@ -263,7 +252,9 @@ function install() { echo "-------------------------" echo "Run the bot !" echo "-------------------------" - echo "You can now use the bot by executing 'source .env/bin/activate; freqtrade'." + echo "You can now use the bot by executing 'source .env/bin/activate; freqtrade '." + echo "You can see the list of available bot subcommands by executing 'source .env/bin/activate; freqtrade --help'." + echo "You verify that freqtrade is installed successfully by running 'source .env/bin/activate; freqtrade --version'." } function plot() { diff --git a/tests/commands/__init__.py b/tests/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/commands/test_build_config.py b/tests/commands/test_build_config.py new file mode 100644 index 000000000..d4ebe1de2 --- /dev/null +++ b/tests/commands/test_build_config.py @@ -0,0 +1,116 @@ +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import rapidjson + +from freqtrade.commands.build_config_commands import (ask_user_config, + ask_user_overwrite, + start_new_config, + validate_is_float, + validate_is_int) +from freqtrade.exceptions import OperationalException +from tests.conftest import get_args, log_has_re + + +def test_validate_is_float(): + assert validate_is_float('2.0') + assert validate_is_float('2.1') + assert validate_is_float('0.1') + assert validate_is_float('-0.5') + assert not validate_is_float('-0.5e') + + +def test_validate_is_int(): + assert validate_is_int('2') + assert validate_is_int('6') + assert validate_is_int('-1') + assert validate_is_int('500') + assert not validate_is_int('2.0') + assert not validate_is_int('2.1') + assert not validate_is_int('-2.1') + assert not validate_is_int('-ee') + + +@pytest.mark.parametrize('exchange', ['bittrex', 'binance', 'kraken', 'ftx']) +def test_start_new_config(mocker, caplog, exchange): + wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + unlink_mock = mocker.patch.object(Path, "unlink", MagicMock()) + mocker.patch('freqtrade.commands.build_config_commands.ask_user_overwrite', return_value=True) + + sample_selections = { + 'max_open_trades': 3, + 'stake_currency': 'USDT', + 'stake_amount': 100, + 'fiat_display_currency': 'EUR', + 'ticker_interval': '15m', + 'dry_run': True, + 'exchange_name': exchange, + 'exchange_key': 'sampleKey', + 'exchange_secret': 'Samplesecret', + 'telegram': False, + 'telegram_token': 'asdf1244', + 'telegram_chat_id': '1144444', + } + mocker.patch('freqtrade.commands.build_config_commands.ask_user_config', + return_value=sample_selections) + args = [ + "new-config", + "--config", + "coolconfig.json" + ] + start_new_config(get_args(args)) + + assert log_has_re("Writing config to .*", caplog) + assert wt_mock.call_count == 1 + assert unlink_mock.call_count == 1 + result = rapidjson.loads(wt_mock.call_args_list[0][0][0], + parse_mode=rapidjson.PM_COMMENTS | rapidjson.PM_TRAILING_COMMAS) + assert result['exchange']['name'] == exchange + assert result['ticker_interval'] == '15m' + + +def test_start_new_config_exists(mocker, caplog): + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + mocker.patch('freqtrade.commands.build_config_commands.ask_user_overwrite', return_value=False) + args = [ + "new-config", + "--config", + "coolconfig.json" + ] + with pytest.raises(OperationalException, match=r"Configuration .* already exists\."): + start_new_config(get_args(args)) + + +def test_ask_user_overwrite(mocker): + """ + Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. + """ + prompt_mock = mocker.patch('freqtrade.commands.build_config_commands.prompt', + return_value={'overwrite': False}) + assert not ask_user_overwrite(Path('test.json')) + assert prompt_mock.call_count == 1 + + prompt_mock.reset_mock() + prompt_mock = mocker.patch('freqtrade.commands.build_config_commands.prompt', + return_value={'overwrite': True}) + assert ask_user_overwrite(Path('test.json')) + assert prompt_mock.call_count == 1 + + +def test_ask_user_config(mocker): + """ + Once https://github.com/tmbo/questionary/issues/35 is implemented, improve this test. + """ + prompt_mock = mocker.patch('freqtrade.commands.build_config_commands.prompt', + return_value={'overwrite': False}) + answers = ask_user_config() + assert isinstance(answers, dict) + assert prompt_mock.call_count == 1 + + prompt_mock = mocker.patch('freqtrade.commands.build_config_commands.prompt', + return_value={}) + + with pytest.raises(OperationalException, match=r"User interrupted interactive questions\."): + ask_user_config() diff --git a/tests/test_utils.py b/tests/commands/test_commands.py similarity index 64% rename from tests/test_utils.py rename to tests/commands/test_commands.py index 40ca9ac02..46350beff 100644 --- a/tests/test_utils.py +++ b/tests/commands/test_commands.py @@ -4,15 +4,19 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade import OperationalException +from freqtrade.commands import (start_convert_data, start_create_userdir, + start_download_data, start_hyperopt_list, + start_hyperopt_show, start_list_exchanges, + start_list_hyperopts, start_list_markets, + start_list_strategies, start_list_timeframes, + start_new_hyperopt, start_new_strategy, + start_show_trades, start_test_pairlist, + start_trading) +from freqtrade.configuration import setup_utils_configuration +from freqtrade.exceptions import OperationalException from freqtrade.state import RunMode -from freqtrade.utils import (setup_utils_configuration, start_create_userdir, - start_download_data, start_list_exchanges, - start_list_markets, start_list_timeframes, - start_new_hyperopt, start_new_strategy, - start_test_pairlist, start_trading, - start_hyperopt_list, start_hyperopt_show) -from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, +from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, + patch_exchange, patched_configuration_load_config_file) @@ -28,7 +32,7 @@ def test_setup_utils_configuration(): assert config['exchange']['secret'] == '' -def test_start_trading_fail(mocker): +def test_start_trading_fail(mocker, caplog): mocker.patch("freqtrade.worker.Worker.run", MagicMock(side_effect=OperationalException)) @@ -39,16 +43,15 @@ def test_start_trading_fail(mocker): 'trade', '-c', 'config.json.example' ] - with pytest.raises(OperationalException): - start_trading(get_args(args)) + start_trading(get_args(args)) assert exitmock.call_count == 1 exitmock.reset_mock() - + caplog.clear() mocker.patch("freqtrade.worker.Worker.__init__", MagicMock(side_effect=OperationalException)) - with pytest.raises(OperationalException): - start_trading(get_args(args)) + start_trading(get_args(args)) assert exitmock.call_count == 0 + assert log_has('Fatal exception!', caplog) def test_list_exchanges(capsys): @@ -215,8 +218,9 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 9 active markets: " - "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XLTCUSDT, XRP/BTC.\n" + assert ("Exchange Bittrex has 10 active markets: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, NEO/BTC, " + "TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) patch_exchange(mocker, api_mock=api_mock, id="binance") @@ -229,7 +233,7 @@ def test_list_markets(mocker, markets, capsys): pargs['config'] = None start_list_markets(pargs, False) captured = capsys.readouterr() - assert re.match("\nExchange Binance has 9 active markets:\n", + assert re.match("\nExchange Binance has 10 active markets:\n", captured.out) patch_exchange(mocker, api_mock=api_mock, id="bittrex") @@ -241,8 +245,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 11 markets: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, LTC/USDT, NEO/BTC, " + assert ("Exchange Bittrex has 12 markets: " + "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, LTC/USDT, NEO/BTC, " "TKN/BTC, XLTCUSDT, XRP/BTC.\n" in captured.out) @@ -254,8 +258,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 8 active pairs: " - "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, NEO/BTC, TKN/BTC, XRP/BTC.\n" + assert ("Exchange Bittrex has 9 active pairs: " + "BLK/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, NEO/BTC, TKN/BTC, XRP/BTC.\n" in captured.out) # Test list-pairs subcommand with --all: all pairs @@ -266,8 +270,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), True) captured = capsys.readouterr() - assert ("Exchange Bittrex has 10 pairs: " - "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, LTC/USDT, NEO/BTC, " + assert ("Exchange Bittrex has 11 pairs: " + "BLK/BTC, BTT/BTC, ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, LTC/USDT, NEO/BTC, " "TKN/BTC, XRP/BTC.\n" in captured.out) @@ -280,8 +284,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 5 active markets with ETH, LTC as base currencies: " - "ETH/BTC, ETH/USDT, LTC/BTC, LTC/USD, XLTCUSDT.\n" + assert ("Exchange Bittrex has 6 active markets with ETH, LTC as base currencies: " + "ETH/BTC, ETH/USDT, LTC/BTC, LTC/ETH, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, base=LTC @@ -293,8 +297,8 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 3 active markets with LTC as base currency: " - "LTC/BTC, LTC/USD, XLTCUSDT.\n" + assert ("Exchange Bittrex has 4 active markets with LTC as base currency: " + "LTC/BTC, LTC/ETH, LTC/USD, XLTCUSDT.\n" in captured.out) # active markets, quote=USDT, USD @@ -382,7 +386,7 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ("Exchange Bittrex has 9 active markets:\n" + assert ("Exchange Bittrex has 10 active markets:\n" in captured.out) # Test tabular output, no markets found @@ -405,7 +409,7 @@ def test_list_markets(mocker, markets, capsys): ] start_list_markets(get_args(args), False) captured = capsys.readouterr() - assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/USD","NEO/BTC",' + assert ('["BLK/BTC","ETH/BTC","ETH/USDT","LTC/BTC","LTC/ETH","LTC/USD","NEO/BTC",' '"TKN/BTC","XLTCUSDT","XRP/BTC"]' in captured.out) @@ -444,11 +448,9 @@ def test_create_datadir_failed(caplog): def test_create_datadir(caplog, mocker): - # Ensure that caplog is empty before starting ... - # Should prevent random failures. - caplog.clear() - cud = mocker.patch("freqtrade.utils.create_userdata_dir", MagicMock()) - csf = mocker.patch("freqtrade.utils.copy_sample_files", MagicMock()) + + cud = mocker.patch("freqtrade.commands.deploy_commands.create_userdata_dir", MagicMock()) + csf = mocker.patch("freqtrade.commands.deploy_commands.copy_sample_files", MagicMock()) args = [ "create-userdir", "--userdir", @@ -458,7 +460,6 @@ def test_create_datadir(caplog, mocker): assert cud.call_count == 1 assert csf.call_count == 1 - assert len(caplog.record_tuples) == 0 def test_start_new_strategy(mocker, caplog): @@ -534,7 +535,7 @@ def test_start_new_hyperopt_no_arg(mocker, caplog): def test_download_data_keyboardInterrupt(mocker, caplog, markets): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(side_effect=KeyboardInterrupt)) patch_exchange(mocker) mocker.patch( @@ -552,7 +553,7 @@ def test_download_data_keyboardInterrupt(mocker, caplog, markets): def test_download_data_no_markets(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker, id='binance') mocker.patch( @@ -570,7 +571,7 @@ def test_download_data_no_markets(mocker, caplog): def test_download_data_no_exchange(mocker, caplog): - mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -590,7 +591,7 @@ def test_download_data_no_pairs(mocker, caplog): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - mocker.patch('freqtrade.utils.refresh_backtest_ohlcv_data', + mocker.patch('freqtrade.commands.data_commands.refresh_backtest_ohlcv_data', MagicMock(return_value=["ETH/BTC", "XRP/BTC"])) patch_exchange(mocker) mocker.patch( @@ -609,9 +610,9 @@ def test_download_data_no_pairs(mocker, caplog): def test_download_data_trades(mocker, caplog): - dl_mock = mocker.patch('freqtrade.utils.refresh_backtest_trades_data', + dl_mock = mocker.patch('freqtrade.commands.data_commands.refresh_backtest_trades_data', MagicMock(return_value=[])) - convert_mock = mocker.patch('freqtrade.utils.convert_trades_to_ohlcv', + convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) mocker.patch( @@ -630,11 +631,75 @@ def test_download_data_trades(mocker, caplog): assert convert_mock.call_count == 1 -def test_start_test_pairlist(mocker, caplog, markets, tickers, default_conf, capsys): +def test_start_list_strategies(mocker, caplog, capsys): + + args = [ + "list-strategies", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy" / "strats"), + "-1" + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_strategies(pargs) + captured = capsys.readouterr() + assert "TestStrategyLegacy" in captured.out + assert "legacy_strategy.py" not in captured.out + assert "DefaultStrategy" in captured.out + + # Test regular output + args = [ + "list-strategies", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy" / "strats"), + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_strategies(pargs) + captured = capsys.readouterr() + assert "TestStrategyLegacy" in captured.out + assert "legacy_strategy.py" in captured.out + assert "DefaultStrategy" in captured.out + + +def test_start_list_hyperopts(mocker, caplog, capsys): + + args = [ + "list-hyperopts", + "--hyperopt-path", + str(Path(__file__).parent.parent / "optimize"), + "-1" + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_hyperopts(pargs) + captured = capsys.readouterr() + assert "TestHyperoptLegacy" not in captured.out + assert "legacy_hyperopt.py" not in captured.out + assert "DefaultHyperOpt" in captured.out + assert "test_hyperopt.py" not in captured.out + + # Test regular output + args = [ + "list-hyperopts", + "--hyperopt-path", + str(Path(__file__).parent.parent / "optimize"), + ] + pargs = get_args(args) + # pargs['config'] = None + start_list_hyperopts(pargs) + captured = capsys.readouterr() + assert "TestHyperoptLegacy" not in captured.out + assert "legacy_hyperopt.py" not in captured.out + assert "DefaultHyperOpt" in captured.out + assert "test_hyperopt.py" in captured.out + + +def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): + patch_exchange(mocker, mock_markets=True) mocker.patch.multiple('freqtrade.exchange.Exchange', - markets=PropertyMock(return_value=markets), exchange_has=MagicMock(return_value=True), - get_tickers=tickers + get_tickers=tickers, ) default_conf['pairlists'] = [ @@ -663,7 +728,7 @@ def test_start_test_pairlist(mocker, caplog, markets, tickers, default_conf, cap assert re.match("['ETH/BTC', 'TKN/BTC', 'BLK/BTC', 'LTC/BTC', 'XRP/BTC']", captured.out) -def test_hyperopt_list(mocker, capsys, hyperopt_results): +def test_hyperopt_list(mocker, capsys, caplog, hyperopt_results): mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.load_previous_results', MagicMock(return_value=hyperopt_results) @@ -709,6 +774,149 @@ def test_hyperopt_list(mocker, capsys, hyperopt_results): assert all(x not in captured.out for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12", "Best result:", "Buy hyperspace params", + "Sell hyperspace params", "ROI table", "Stoploss"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--no-color", + "--min-trades", "20" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 3/12", " 6/12", " 7/12", " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 4/12", " 5/12", " 8/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--max-trades", "20" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--min-avg-profit", "0.11" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 10/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--max-avg-profit", "0.10" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", " 9/12", + " 11/12"]) + assert all(x not in captured.out + for x in [" 2/12", " 4/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--min-total-profit", "0.4" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--max-total-profit", "0.4" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 5/12", " 6/12", " 7/12", " 8/12", + " 9/12", " 11/12"]) + assert all(x not in captured.out + for x in [" 4/12", " 10/12", " 12/12"]) + args = [ + "hyperopt-list", + "--profitable", + "--no-details", + "--min-avg-time", "2000" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 10/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 2/12", " 3/12", " 4/12", " 5/12", " 6/12", " 7/12", + " 8/12", " 9/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--max-avg-time", "1500" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + assert all(x in captured.out + for x in [" 2/12", " 6/12"]) + assert all(x not in captured.out + for x in [" 1/12", " 3/12", " 4/12", " 5/12", " 7/12", " 8/12" + " 9/12", " 10/12", " 11/12", " 12/12"]) + args = [ + "hyperopt-list", + "--no-details", + "--export-csv", "test_file.csv" + ] + pargs = get_args(args) + pargs['config'] = None + start_hyperopt_list(pargs) + captured = capsys.readouterr() + log_has("CSV file created: test_file.csv", caplog) + f = Path("test_file.csv") + assert 'Best,1,2,-1.25%,-0.00125625,,-2.51,"3,930.0 m",0.43662' in f.read_text() + assert f.is_file() + f.unlink() def test_hyperopt_show(mocker, capsys, hyperopt_results): @@ -789,3 +997,90 @@ def test_hyperopt_show(mocker, capsys, hyperopt_results): with pytest.raises(OperationalException, match="The index of the epoch to show should be less than 4."): start_hyperopt_show(pargs) + + +def test_convert_data(mocker, testdatadir): + ohlcv_mock = mocker.patch("freqtrade.commands.data_commands.convert_ohlcv_format") + trades_mock = mocker.patch("freqtrade.commands.data_commands.convert_trades_format") + args = [ + "convert-data", + "--format-from", + "json", + "--format-to", + "jsongz", + "--datadir", + str(testdatadir), + ] + pargs = get_args(args) + pargs['config'] = None + start_convert_data(pargs, True) + assert trades_mock.call_count == 0 + assert ohlcv_mock.call_count == 1 + assert ohlcv_mock.call_args[1]['convert_from'] == 'json' + assert ohlcv_mock.call_args[1]['convert_to'] == 'jsongz' + assert ohlcv_mock.call_args[1]['erase'] is False + + +def test_convert_data_trades(mocker, testdatadir): + ohlcv_mock = mocker.patch("freqtrade.commands.data_commands.convert_ohlcv_format") + trades_mock = mocker.patch("freqtrade.commands.data_commands.convert_trades_format") + args = [ + "convert-trade-data", + "--format-from", + "jsongz", + "--format-to", + "json", + "--datadir", + str(testdatadir), + ] + pargs = get_args(args) + pargs['config'] = None + start_convert_data(pargs, False) + assert ohlcv_mock.call_count == 0 + assert trades_mock.call_count == 1 + assert trades_mock.call_args[1]['convert_from'] == 'jsongz' + assert trades_mock.call_args[1]['convert_to'] == 'json' + assert trades_mock.call_args[1]['erase'] is False + + +@pytest.mark.usefixtures("init_persistence") +def test_show_trades(mocker, fee, capsys, caplog): + mocker.patch("freqtrade.persistence.init") + create_mock_trades(fee) + args = [ + "show-trades", + "--db-url", + "sqlite:///" + ] + pargs = get_args(args) + pargs['config'] = None + start_show_trades(pargs) + assert log_has("Printing 3 Trades: ", caplog) + captured = capsys.readouterr() + assert "Trade(id=1" in captured.out + assert "Trade(id=2" in captured.out + assert "Trade(id=3" in captured.out + args = [ + "show-trades", + "--db-url", + "sqlite:///", + "--print-json", + "--trade-ids", "1", "2" + ] + pargs = get_args(args) + pargs['config'] = None + start_show_trades(pargs) + + captured = capsys.readouterr() + assert log_has("Printing 2 Trades: ", caplog) + assert '"trade_id": 1' in captured.out + assert '"trade_id": 2' in captured.out + assert '"trade_id": 3' not in captured.out + args = [ + "show-trades", + ] + pargs = get_args(args) + pargs['config'] = None + + with pytest.raises(OperationalException, match=r"--db-url is required for this command."): + start_show_trades(pargs) diff --git a/tests/conftest.py b/tests/conftest.py index 82111528e..d15cba1de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,8 +14,8 @@ import pytest from telegram import Chat, Message, Update from freqtrade import constants, persistence -from freqtrade.configuration import Arguments -from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.commands import Arguments +from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot @@ -60,8 +60,10 @@ def patch_exchange(mocker, api_mock=None, id='bittrex', mock_markets=True) -> No mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.validate_ordertypes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency', MagicMock()) mocker.patch('freqtrade.exchange.Exchange.id', PropertyMock(return_value=id)) mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value=id.title())) + mocker.patch('freqtrade.exchange.Exchange.precisionMode', PropertyMock(return_value=2)) if mock_markets: mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=get_markets())) @@ -77,7 +79,7 @@ def get_patched_exchange(mocker, config, api_mock=None, id='bittrex', patch_exchange(mocker, api_mock, id, mock_markets) config["exchange"]["name"] = id try: - exchange = ExchangeResolver(id, config).exchange + exchange = ExchangeResolver.load_exchange(id, config) except ImportError: exchange = Exchange(config) return exchange @@ -164,24 +166,70 @@ def patch_get_signal(freqtrade: FreqtradeBot, value=(True, False)) -> None: freqtrade.exchange.refresh_latest_ohlcv = lambda p: None -@pytest.fixture(autouse=True) -def patch_coinmarketcap(mocker) -> None: +def create_mock_trades(fee): """ - Mocker to coinmarketcap to speed up tests - :param mocker: mocker to patch coinmarketcap class + Create some fake trades ... + """ + # Simulate dry_run entries + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='dry_run_buy_12345' + ) + Trade.session.add(trade) + + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + close_rate=0.128, + close_profit=0.005, + exchange='bittrex', + is_open=False, + open_order_id='dry_run_sell_12345' + ) + Trade.session.add(trade) + + # Simulate prod entry + trade = Trade( + pair='ETC/BTC', + stake_amount=0.001, + amount=123.0, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_rate=0.123, + exchange='bittrex', + open_order_id='prod_buy_12345' + ) + Trade.session.add(trade) + + +@pytest.fixture(autouse=True) +def patch_coingekko(mocker) -> None: + """ + Mocker to coingekko to speed up tests + :param mocker: mocker to patch coingekko class :return: None """ - tickermock = MagicMock(return_value={'price_usd': 12345.0}) - listmock = MagicMock(return_value={'data': [{'id': 1, 'name': 'Bitcoin', 'symbol': 'BTC', - 'website_slug': 'bitcoin'}, - {'id': 1027, 'name': 'Ethereum', 'symbol': 'ETH', - 'website_slug': 'ethereum'} - ]}) + tickermock = MagicMock(return_value={'bitcoin': {'usd': 12345.0}, 'ethereum': {'usd': 12345.0}}) + listmock = MagicMock(return_value=[{'id': 'bitcoin', 'name': 'Bitcoin', 'symbol': 'btc', + 'website_slug': 'bitcoin'}, + {'id': 'ethereum', 'name': 'Ethereum', 'symbol': 'eth', + 'website_slug': 'ethereum'} + ]) mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.Market', - ticker=tickermock, - listings=listmock, + 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', + get_price=tickermock, + get_coins_list=listmock, ) @@ -201,6 +249,7 @@ def default_conf(testdatadir): "fiat_display_currency": "USD", "ticker_interval": '5m', "dry_run": True, + "cancel_open_orders_on_exit": False, "minimal_roi": { "40": 0.0, "30": 0.01, @@ -255,7 +304,9 @@ def default_conf(testdatadir): "db_url": "sqlite://", "user_data_dir": Path("user_data"), "verbosity": 3, - "strategy": "DefaultStrategy" + "strategy_path": str(Path(__file__).parent / "strategy" / "strats"), + "strategy": "DefaultStrategy", + "internals": {}, } return configuration @@ -572,7 +623,34 @@ def get_markets(): } }, 'info': {}, - } + }, + 'LTC/ETH': { + 'id': 'LTCETH', + 'symbol': 'LTC/ETH', + 'base': 'LTC', + 'quote': 'ETH', + 'active': True, + 'precision': { + 'base': 8, + 'quote': 8, + 'amount': 3, + 'price': 5 + }, + 'limits': { + 'amount': { + 'min': 0.001, + 'max': 10000000.0 + }, + 'price': { + 'min': 1e-05, + 'max': 1000.0 + }, + 'cost': { + 'min': 0.01, + 'max': None + } + }, + }, } @@ -638,6 +716,56 @@ def shitcoinmarkets(markets): }, 'info': {}, }, + 'NANO/USDT': { + "percentage": True, + "tierBased": False, + "taker": 0.001, + "maker": 0.001, + "precision": { + "base": 8, + "quote": 8, + "amount": 2, + "price": 4 + }, + "limits": { + }, + "id": "NANOUSDT", + "symbol": "NANO/USDT", + "base": "NANO", + "quote": "USDT", + "baseId": "NANO", + "quoteId": "USDT", + "info": {}, + "type": "spot", + "spot": True, + "future": False, + "active": True + }, + 'ADAHALF/USDT': { + "percentage": True, + "tierBased": False, + "taker": 0.001, + "maker": 0.001, + "precision": { + "base": 8, + "quote": 8, + "amount": 2, + "price": 4 + }, + "limits": { + }, + "id": "ADAHALFUSDT", + "symbol": "ADAHALF/USDT", + "base": "ADAHALF", + "quote": "USDT", + "baseId": "ADAHALF", + "quoteId": "USDT", + "info": {}, + "type": "spot", + "spot": True, + "future": False, + "active": True + }, }) return shitmarkets @@ -653,10 +781,11 @@ def limit_buy_order(): 'id': 'mocked_limit_buy', 'type': 'limit', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'price': 0.00001099, 'amount': 90.99181073, + 'filled': 90.99181073, 'remaining': 0.0, 'status': 'closed' } @@ -668,10 +797,11 @@ def market_buy_order(): 'id': 'mocked_market_buy', 'type': 'market', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'price': 0.00004099, 'amount': 91.99181073, + 'filled': 91.99181073, 'remaining': 0.0, 'status': 'closed' } @@ -683,10 +813,11 @@ def market_sell_order(): 'id': 'mocked_limit_sell', 'type': 'market', 'side': 'sell', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': arrow.utcnow().isoformat(), 'price': 0.00004173, 'amount': 91.99181073, + 'filled': 91.99181073, 'remaining': 0.0, 'status': 'closed' } @@ -698,10 +829,11 @@ def limit_buy_order_old(): 'id': 'mocked_limit_buy_old', 'type': 'limit', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': str(arrow.utcnow().shift(minutes=-601).datetime), 'price': 0.00001099, 'amount': 90.99181073, + 'filled': 0.0, 'remaining': 90.99181073, 'status': 'open' } @@ -713,10 +845,11 @@ def limit_sell_order_old(): 'id': 'mocked_limit_sell_old', 'type': 'limit', 'side': 'sell', - 'pair': 'ETH/BTC', + 'symbol': 'ETH/BTC', 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, + 'filled': 0.0, 'remaining': 90.99181073, 'status': 'open' } @@ -728,10 +861,11 @@ def limit_buy_order_old_partial(): 'id': 'mocked_limit_buy_old_partial', 'type': 'limit', 'side': 'buy', - 'pair': 'ETH/BTC', + 'symbol': 'ETH/BTC', 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), 'price': 0.00001099, 'amount': 90.99181073, + 'filled': 23.0, 'remaining': 67.99181073, 'status': 'open' } @@ -741,10 +875,103 @@ def limit_buy_order_old_partial(): def limit_buy_order_old_partial_canceled(limit_buy_order_old_partial): res = deepcopy(limit_buy_order_old_partial) res['status'] = 'canceled' - res['fee'] = {'cost': 0.0001, 'currency': 'ETH'} + res['fee'] = {'cost': 0.023, 'currency': 'ETH'} return res +@pytest.fixture(scope='function') +def limit_buy_order_canceled_empty(request): + # Indirect fixture + # Documentation: + # https://docs.pytest.org/en/latest/example/parametrize.html#apply-indirect-on-particular-arguments + + exchange_name = request.param + if exchange_name == 'ftx': + return { + 'info': {}, + 'id': '1234512345', + 'clientOrderId': None, + 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'lastTradeTimestamp': None, + 'symbol': 'LTC/USDT', + 'type': 'limit', + 'side': 'buy', + 'price': 34.3225, + 'amount': 0.55, + 'cost': 0.0, + 'average': None, + 'filled': 0.0, + 'remaining': 0.0, + 'status': 'closed', + 'fee': None, + 'trades': None + } + elif exchange_name == 'kraken': + return { + 'info': {}, + 'id': 'AZNPFF-4AC4N-7MKTAT', + 'clientOrderId': None, + 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'lastTradeTimestamp': None, + 'status': 'canceled', + 'symbol': 'LTC/USDT', + 'type': 'limit', + 'side': 'buy', + 'price': 34.3225, + 'cost': 0.0, + 'amount': 0.55, + 'filled': 0.0, + 'average': 0.0, + 'remaining': 0.55, + 'fee': {'cost': 0.0, 'rate': None, 'currency': 'USDT'}, + 'trades': [] + } + elif exchange_name == 'binance': + return { + 'info': {}, + 'id': '1234512345', + 'clientOrderId': 'alb1234123', + 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'lastTradeTimestamp': None, + 'symbol': 'LTC/USDT', + 'type': 'limit', + 'side': 'buy', + 'price': 0.016804, + 'amount': 0.55, + 'cost': 0.0, + 'average': None, + 'filled': 0.0, + 'remaining': 0.55, + 'status': 'canceled', + 'fee': None, + 'trades': None + } + else: + return { + 'info': {}, + 'id': '1234512345', + 'clientOrderId': 'alb1234123', + 'timestamp': arrow.utcnow().shift(minutes=-601).timestamp, + 'datetime': arrow.utcnow().shift(minutes=-601).isoformat(), + 'lastTradeTimestamp': None, + 'symbol': 'LTC/USDT', + 'type': 'limit', + 'side': 'buy', + 'price': 0.016804, + 'amount': 0.55, + 'cost': 0.0, + 'average': None, + 'filled': 0.0, + 'remaining': 0.55, + 'status': 'canceled', + 'fee': None, + 'trades': None + } + + @pytest.fixture def limit_sell_order(): return { @@ -755,6 +982,7 @@ def limit_sell_order(): 'datetime': arrow.utcnow().isoformat(), 'price': 0.00001173, 'amount': 90.99181073, + 'filled': 90.99181073, 'remaining': 0.0, 'status': 'closed' } @@ -794,15 +1022,15 @@ def order_book_l2(): @pytest.fixture -def ticker_history_list(): +def ohlcv_history_list(): return [ [ 1511686200000, # unix timestamp ms - 8.794e-05, # open - 8.948e-05, # high - 8.794e-05, # low - 8.88e-05, # close - 0.0877869, # volume (in quote currency) + 8.794e-05, # open + 8.948e-05, # high + 8.794e-05, # low + 8.88e-05, # close + 0.0877869, # volume (in quote currency) ], [ 1511686500000, @@ -824,8 +1052,9 @@ def ticker_history_list(): @pytest.fixture -def ticker_history(ticker_history_list): - return parse_ticker_dataframe(ticker_history_list, "5m", pair="UNITTEST/BTC", fill_missing=True) +def ohlcv_history(ohlcv_history_list): + return ohlcv_to_dataframe(ohlcv_history_list, "5m", pair="UNITTEST/BTC", + fill_missing=True) @pytest.fixture @@ -1112,14 +1341,59 @@ def tickers(): 'quoteVolume': 1154.19266394, 'info': {} }, + "NANO/USDT": { + "symbol": "NANO/USDT", + "timestamp": 1580469388244, + "datetime": "2020-01-31T11:16:28.244Z", + "high": 0.7519, + "low": 0.7154, + "bid": 0.7305, + "bidVolume": 300.3, + "ask": 0.7342, + "askVolume": 15.14, + "vwap": 0.73645591, + "open": 0.7154, + "close": 0.7342, + "last": 0.7342, + "previousClose": 0.7189, + "change": 0.0188, + "percentage": 2.628, + "average": None, + "baseVolume": 439472.44, + "quoteVolume": 323652.075405, + "info": {} + }, + # Example of leveraged pair with incomplete info + "ADAHALF/USDT": { + "symbol": "ADAHALF/USDT", + "timestamp": 1580469388244, + "datetime": "2020-01-31T11:16:28.244Z", + "high": None, + "low": None, + "bid": 0.7305, + "bidVolume": None, + "ask": 0.7342, + "askVolume": None, + "vwap": None, + "open": None, + "close": None, + "last": None, + "previousClose": None, + "change": None, + "percentage": 2.628, + "average": None, + "baseVolume": 0.0, + "quoteVolume": 0.0, + "info": {} + }, }) @pytest.fixture def result(testdatadir): with (testdatadir / 'UNITTEST_BTC-1m.json').open('r') as data_file: - return parse_ticker_dataframe(json.load(data_file), '1m', pair="UNITTEST/BTC", - fill_missing=True) + return ohlcv_to_dataframe(json.load(data_file), '1m', pair="UNITTEST/BTC", + fill_missing=True) @pytest.fixture(scope="function") @@ -1149,6 +1423,15 @@ def trades_for_order(): @pytest.fixture(scope="function") def trades_history(): + return [[1565798399463, '126181329', None, 'buy', 0.019627, 0.04, 0.00078508], + [1565798399629, '126181330', None, 'buy', 0.019627, 0.244, 0.004788987999999999], + [1565798399752, '126181331', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], + [1565798399862, '126181332', None, 'sell', 0.019626, 0.011, 0.00021588599999999999], + [1565798399872, '126181333', None, 'sell', 0.019626, 0.011, 0.00021588599999999999]] + + +@pytest.fixture(scope="function") +def fetch_trades_result(): return [{'info': {'a': 126181329, 'p': '0.01962700', 'q': '0.04000000', @@ -1303,7 +1586,7 @@ def buy_order_fee(): 'id': 'mocked_limit_buy_old', 'type': 'limit', 'side': 'buy', - 'pair': 'mocked', + 'symbol': 'mocked', 'datetime': str(arrow.utcnow().shift(minutes=-601).datetime), 'price': 0.245441, 'amount': 8.0, @@ -1317,12 +1600,12 @@ def buy_order_fee(): def edge_conf(default_conf): conf = deepcopy(default_conf) conf['max_open_trades'] = -1 + conf['tradable_balance_ratio'] = 0.5 conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT conf['edge'] = { "enabled": True, "process_throttle_secs": 1800, "calculate_since_number_of_days": 14, - "capital_available_percentage": 0.5, "allowed_risk": 0.01, "stoploss_range_min": -0.01, "stoploss_range_max": -0.1, @@ -1422,7 +1705,7 @@ def hyperopt_results(): { 'loss': 0.4366182531161519, 'params_dict': { - 'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1190, 'roi_t2': 541, 'roi_t3': 408, 'roi_p1': 0.026035863879169705, 'roi_p2': 0.12508730043628782, 'roi_p3': 0.27766427921605896, 'stoploss': -0.2562930402099556}, # noqa: E501 + 'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1190, 'roi_t2': 541, 'roi_t3': 408, 'roi_p1': 0.026035863879169705, 'roi_p2': 0.12508730043628782, 'roi_p3': 0.27766427921605896, 'stoploss': -0.2562930402099556}, # noqa: E501 'params_details': {'buy': {'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4287874435315165, 408: 0.15112316431545753, 949: 0.026035863879169705, 2139: 0}, 'stoploss': {'stoploss': -0.2562930402099556}}, # noqa: E501 'results_metrics': {'trade_count': 2, 'avg_profit': -1.254995, 'total_profit': -0.00125625, 'profit': -2.50999, 'duration': 3930.0}, # noqa: E501 'results_explanation': ' 2 trades. Avg profit -1.25%. Total profit -0.00125625 BTC ( -2.51Σ%). Avg duration 3930.0 min.', # noqa: E501 @@ -1433,11 +1716,12 @@ def hyperopt_results(): }, { 'loss': 20.0, 'params_dict': { - 'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 334, 'roi_t2': 683, 'roi_t3': 140, 'roi_p1': 0.06403981740598495, 'roi_p2': 0.055519840060645045, 'roi_p3': 0.3253712811342459, 'stoploss': -0.338070047333259}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, # noqa: E501 - 'sell': {'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, # noqa: E501 - 'roi': {0: 0.4449309386008759, 140: 0.11955965746663, 823: 0.06403981740598495, 1157: 0}, # noqa: E501 - 'stoploss': {'stoploss': -0.338070047333259}}, + 'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal', 'roi_t1': 334, 'roi_t2': 683, 'roi_t3': 140, 'roi_p1': 0.06403981740598495, 'roi_p2': 0.055519840060645045, 'roi_p3': 0.3253712811342459, 'stoploss': -0.338070047333259}, # noqa: E501 + 'params_details': { + 'buy': {'mfi-value': 17, 'fastd-value': 38, 'adx-value': 48, 'rsi-value': 22, 'mfi-enabled': True, 'fastd-enabled': False, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, # noqa: E501 + 'sell': {'sell-mfi-value': 96, 'sell-fastd-value': 68, 'sell-adx-value': 63, 'sell-rsi-value': 81, 'sell-mfi-enabled': False, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-sar_reversal'}, # noqa: E501 + 'roi': {0: 0.4449309386008759, 140: 0.11955965746663, 823: 0.06403981740598495, 1157: 0}, # noqa: E501 + 'stoploss': {'stoploss': -0.338070047333259}}, 'results_metrics': {'trade_count': 1, 'avg_profit': 0.12357, 'total_profit': 6.185e-05, 'profit': 0.12357, 'duration': 1200.0}, # noqa: E501 'results_explanation': ' 1 trades. Avg profit 0.12%. Total profit 0.00006185 BTC ( 0.12Σ%). Avg duration 1200.0 min.', # noqa: E501 'total_profit': 6.185e-05, @@ -1484,8 +1768,9 @@ def hyperopt_results(): }, { 'loss': 4.713497421432944, 'params_dict': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower', 'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal', 'roi_t1': 771, 'roi_t2': 620, 'roi_t3': 145, 'roi_p1': 0.0586919200378493, 'roi_p2': 0.04984118697312542, 'roi_p3': 0.37521058680247044, 'stoploss': -0.14613268022709905}, # noqa: E501 - 'params_details': {'buy': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.4837436938134452, 145: 0.10853310701097472, 765: 0.0586919200378493, 1536: 0}, # noqa: E501 - 'stoploss': {'stoploss': -0.14613268022709905}}, # noqa: E501 + 'params_details': { + 'buy': {'mfi-value': 13, 'fastd-value': 41, 'adx-value': 21, 'rsi-value': 29, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': False, 'rsi-enabled': False, 'trigger': 'bb_lower'}, 'sell': {'sell-mfi-value': 99, 'sell-fastd-value': 60, 'sell-adx-value': 81, 'sell-rsi-value': 69, 'sell-mfi-enabled': True, 'sell-fastd-enabled': True, 'sell-adx-enabled': True, 'sell-rsi-enabled': False, 'sell-trigger': 'sell-macd_cross_signal'}, 'roi': {0: 0.4837436938134452, 145: 0.10853310701097472, 765: 0.0586919200378493, 1536: 0}, # noqa: E501 + 'stoploss': {'stoploss': -0.14613268022709905}}, # noqa: E501 'results_metrics': {'trade_count': 318, 'avg_profit': -0.39833954716981146, 'total_profit': -0.06339929, 'profit': -126.67197600000004, 'duration': 3140.377358490566}, # noqa: E501 'results_explanation': ' 318 trades. Avg profit -0.40%. Total profit -0.06339929 BTC (-126.67Σ%). Avg duration 3140.4 min.', # noqa: E501 'total_profit': -0.06339929, diff --git a/tests/data/test_btanalysis.py b/tests/data/test_btanalysis.py index 13711c63e..4da2acc5e 100644 --- a/tests/data/test_btanalysis.py +++ b/tests/data/test_btanalysis.py @@ -1,18 +1,21 @@ +from pathlib import Path from unittest.mock import MagicMock import pytest from arrow import Arrow -from pandas import DataFrame, DateOffset, to_datetime +from pandas import DataFrame, DateOffset, Timestamp, to_datetime from freqtrade.configuration import TimeRange from freqtrade.data.btanalysis import (BT_DATA_COLUMNS, - combine_tickers_with_mean, + analyze_trade_parallelism, + calculate_max_drawdown, + combine_dataframes_with_mean, create_cum_profit, extract_trades_of_period, load_backtest_data, load_trades, - load_trades_from_db, analyze_trade_parallelism) + load_trades_from_db) from freqtrade.data.history import load_data, load_pair_history -from tests.test_persistence import create_mock_trades +from tests.conftest import create_mock_trades def test_load_backtest_data(testdatadir): @@ -20,7 +23,7 @@ def test_load_backtest_data(testdatadir): filename = testdatadir / "backtest-result_test.json" bt_data = load_backtest_data(filename) assert isinstance(bt_data, DataFrame) - assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profitabs"] + assert list(bt_data.columns) == BT_DATA_COLUMNS + ["profit"] assert len(bt_data) == 179 # Test loading from string (must yield same result) @@ -102,6 +105,7 @@ def test_load_trades(default_conf, mocker): load_trades("DB", db_url=default_conf.get('db_url'), exportfilename=default_conf.get('exportfilename'), + no_trades=False ) assert db_mock.call_count == 1 @@ -109,22 +113,32 @@ def test_load_trades(default_conf, mocker): db_mock.reset_mock() bt_mock.reset_mock() - default_conf['exportfilename'] = "testfile.json" + default_conf['exportfilename'] = Path("testfile.json") load_trades("file", db_url=default_conf.get('db_url'), - exportfilename=default_conf.get('exportfilename'),) + exportfilename=default_conf.get('exportfilename'), + ) assert db_mock.call_count == 0 assert bt_mock.call_count == 1 + db_mock.reset_mock() + bt_mock.reset_mock() + default_conf['exportfilename'] = "testfile.json" + load_trades("file", + db_url=default_conf.get('db_url'), + exportfilename=default_conf.get('exportfilename'), + no_trades=True + ) -def test_combine_tickers_with_mean(testdatadir): + assert db_mock.call_count == 0 + assert bt_mock.call_count == 0 + + +def test_combine_dataframes_with_mean(testdatadir): pairs = ["ETH/BTC", "ADA/BTC"] - tickers = load_data(datadir=testdatadir, - pairs=pairs, - timeframe='5m' - ) - df = combine_tickers_with_mean(tickers) + data = load_data(datadir=testdatadir, pairs=pairs, timeframe='5m') + df = combine_dataframes_with_mean(data) assert isinstance(df, DataFrame) assert "ETH/BTC" in df.columns assert "ADA/BTC" in df.columns @@ -163,3 +177,42 @@ def test_create_cum_profit1(testdatadir): assert "cum_profits" in cum_profits.columns assert cum_profits.iloc[0]['cum_profits'] == 0 assert cum_profits.iloc[-1]['cum_profits'] == 0.0798005 + + +def test_calculate_max_drawdown(testdatadir): + filename = testdatadir / "backtest-result_test.json" + bt_data = load_backtest_data(filename) + drawdown, h, low = calculate_max_drawdown(bt_data) + assert isinstance(drawdown, float) + assert pytest.approx(drawdown) == 0.21142322 + assert isinstance(h, Timestamp) + assert isinstance(low, Timestamp) + assert h == Timestamp('2018-01-24 14:25:00', tz='UTC') + assert low == Timestamp('2018-01-30 04:45:00', tz='UTC') + with pytest.raises(ValueError, match='Trade dataframe empty.'): + drawdown, h, low = calculate_max_drawdown(DataFrame()) + + +def test_calculate_max_drawdown2(): + values = [0.011580, 0.010048, 0.011340, 0.012161, 0.010416, 0.010009, 0.020024, + -0.024662, -0.022350, 0.020496, -0.029859, -0.030511, 0.010041, 0.010872, + -0.025782, 0.010400, 0.012374, 0.012467, 0.114741, 0.010303, 0.010088, + -0.033961, 0.010680, 0.010886, -0.029274, 0.011178, 0.010693, 0.010711] + + dates = [Arrow(2020, 1, 1).shift(days=i) for i in range(len(values))] + df = DataFrame(zip(values, dates), columns=['profit', 'open_time']) + # sort by profit and reset index + df = df.sort_values('profit').reset_index(drop=True) + df1 = df.copy() + drawdown, h, low = calculate_max_drawdown(df, date_col='open_time', value_col='profit') + # Ensure df has not been altered. + assert df.equals(df1) + + assert isinstance(drawdown, float) + # High must be before low + assert h < low + assert drawdown == 0.091755 + + df = DataFrame(zip(values[:5], dates[:5]), columns=['profit', 'open_time']) + with pytest.raises(ValueError, match='No losing trade, therefore no drawdown.'): + calculate_max_drawdown(df, date_col='open_time', value_col='profit') diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 414551c95..4a580366f 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -1,24 +1,31 @@ # pragma pylint: disable=missing-docstring, C0103 import logging -from freqtrade.data.converter import parse_ticker_dataframe, ohlcv_fill_up_missing_data -from freqtrade.data.history import load_pair_history, validate_backtest_data, get_timerange +from freqtrade.configuration.timerange import TimeRange +from freqtrade.data.converter import (convert_ohlcv_format, + convert_trades_format, + ohlcv_fill_up_missing_data, + ohlcv_to_dataframe, trades_dict_to_list, + trades_remove_duplicates, trim_dataframe) +from freqtrade.data.history import (get_timerange, load_data, + load_pair_history, validate_backtest_data) from tests.conftest import log_has +from tests.data.test_history import _backup_file, _clean_test_file def test_dataframe_correct_columns(result): assert result.columns.tolist() == ['date', 'open', 'high', 'low', 'close', 'volume'] -def test_parse_ticker_dataframe(ticker_history_list, caplog): +def test_ohlcv_to_dataframe(ohlcv_history_list, caplog): columns = ['date', 'open', 'high', 'low', 'close', 'volume'] caplog.set_level(logging.DEBUG) # Test file with BV data - dataframe = parse_ticker_dataframe(ticker_history_list, '5m', - pair="UNITTEST/BTC", fill_missing=True) + dataframe = ohlcv_to_dataframe(ohlcv_history_list, '5m', pair="UNITTEST/BTC", + fill_missing=True) assert dataframe.columns.tolist() == columns - assert log_has('Parsing tickerlist to dataframe', caplog) + assert log_has('Converting candle (OHLCV) data to dataframe for pair UNITTEST/BTC.', caplog) def test_ohlcv_fill_up_missing_data(testdatadir, caplog): @@ -78,7 +85,8 @@ def test_ohlcv_fill_up_missing_data2(caplog): ] # Generate test-data without filling missing - data = parse_ticker_dataframe(ticks, timeframe, pair="UNITTEST/BTC", fill_missing=False) + data = ohlcv_to_dataframe(ticks, timeframe, pair="UNITTEST/BTC", + fill_missing=False) assert len(data) == 3 caplog.set_level(logging.DEBUG) data2 = ohlcv_fill_up_missing_data(data, timeframe, "UNITTEST/BTC") @@ -134,14 +142,152 @@ def test_ohlcv_drop_incomplete(caplog): ] ] caplog.set_level(logging.DEBUG) - data = parse_ticker_dataframe(ticks, timeframe, pair="UNITTEST/BTC", - fill_missing=False, drop_incomplete=False) + data = ohlcv_to_dataframe(ticks, timeframe, pair="UNITTEST/BTC", + fill_missing=False, drop_incomplete=False) assert len(data) == 4 assert not log_has("Dropping last candle", caplog) # Drop last candle - data = parse_ticker_dataframe(ticks, timeframe, pair="UNITTEST/BTC", - fill_missing=False, drop_incomplete=True) + data = ohlcv_to_dataframe(ticks, timeframe, pair="UNITTEST/BTC", + fill_missing=False, drop_incomplete=True) assert len(data) == 3 assert log_has("Dropping last candle", caplog) + + +def test_trim_dataframe(testdatadir) -> None: + data = load_data( + datadir=testdatadir, + timeframe='1m', + pairs=['UNITTEST/BTC'] + )['UNITTEST/BTC'] + min_date = int(data.iloc[0]['date'].timestamp()) + max_date = int(data.iloc[-1]['date'].timestamp()) + data_modify = data.copy() + + # Remove first 30 minutes (1800 s) + tr = TimeRange('date', None, min_date + 1800, 0) + data_modify = trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 30 + assert all(data_modify.iloc[-1] == data.iloc[-1]) + assert all(data_modify.iloc[0] == data.iloc[30]) + + data_modify = data.copy() + # Remove last 30 minutes (1800 s) + tr = TimeRange(None, 'date', 0, max_date - 1800) + data_modify = trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 30 + assert all(data_modify.iloc[0] == data.iloc[0]) + assert all(data_modify.iloc[-1] == data.iloc[-31]) + + data_modify = data.copy() + # Remove first 25 and last 30 minutes (1800 s) + tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800) + data_modify = trim_dataframe(data_modify, tr) + assert not data_modify.equals(data) + assert len(data_modify) < len(data) + assert len(data_modify) == len(data) - 55 + # first row matches 25th original row + assert all(data_modify.iloc[0] == data.iloc[25]) + + +def test_trades_remove_duplicates(trades_history): + trades_history1 = trades_history * 3 + assert len(trades_history1) == len(trades_history) * 3 + res = trades_remove_duplicates(trades_history1) + assert len(res) == len(trades_history) + for i, t in enumerate(res): + assert t == trades_history[i] + + +def test_trades_dict_to_list(fetch_trades_result): + res = trades_dict_to_list(fetch_trades_result) + assert isinstance(res, list) + assert isinstance(res[0], list) + for i, t in enumerate(res): + assert t[0] == fetch_trades_result[i]['timestamp'] + assert t[1] == fetch_trades_result[i]['id'] + assert t[2] == fetch_trades_result[i]['type'] + assert t[3] == fetch_trades_result[i]['side'] + assert t[4] == fetch_trades_result[i]['price'] + assert t[5] == fetch_trades_result[i]['amount'] + assert t[6] == fetch_trades_result[i]['cost'] + + +def test_convert_trades_format(mocker, default_conf, testdatadir): + files = [{'old': testdatadir / "XRP_ETH-trades.json.gz", + 'new': testdatadir / "XRP_ETH-trades.json"}, + {'old': testdatadir / "XRP_OLD-trades.json.gz", + 'new': testdatadir / "XRP_OLD-trades.json"}, + ] + for file in files: + _backup_file(file['old'], copy_file=True) + assert not file['new'].exists() + + default_conf['datadir'] = testdatadir + + convert_trades_format(default_conf, convert_from='jsongz', + convert_to='json', erase=False) + + for file in files: + assert file['new'].exists() + assert file['old'].exists() + + # Remove original file + file['old'].unlink() + # Convert back + convert_trades_format(default_conf, convert_from='json', + convert_to='jsongz', erase=True) + for file in files: + assert file['old'].exists() + assert not file['new'].exists() + + _clean_test_file(file['old']) + if file['new'].exists(): + file['new'].unlink() + + +def test_convert_ohlcv_format(mocker, default_conf, testdatadir): + file1 = testdatadir / "XRP_ETH-5m.json" + file1_new = testdatadir / "XRP_ETH-5m.json.gz" + file2 = testdatadir / "XRP_ETH-1m.json" + file2_new = testdatadir / "XRP_ETH-1m.json.gz" + _backup_file(file1, copy_file=True) + _backup_file(file2, copy_file=True) + default_conf['datadir'] = testdatadir + default_conf['pairs'] = ['XRP_ETH'] + default_conf['timeframes'] = ['1m', '5m'] + + assert not file1_new.exists() + assert not file2_new.exists() + + convert_ohlcv_format(default_conf, convert_from='json', + convert_to='jsongz', erase=False) + + assert file1_new.exists() + assert file2_new.exists() + assert file1.exists() + assert file2.exists() + + # Remove original files + file1.unlink() + file2.unlink() + # Convert back + convert_ohlcv_format(default_conf, convert_from='jsongz', + convert_to='json', erase=True) + + assert file1.exists() + assert file2.exists() + assert not file1_new.exists() + assert not file2_new.exists() + + _clean_test_file(file1) + _clean_test_file(file2) + if file1_new.exists(): + file1_new.unlink() + if file2_new.exists(): + file2_new.unlink() diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 1dbe20936..c2d6e82f1 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -1,25 +1,28 @@ from unittest.mock import MagicMock from pandas import DataFrame +import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.pairlist.pairlistmanager import PairListManager +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.state import RunMode from tests.conftest import get_patched_exchange -def test_ohlcv(mocker, default_conf, ticker_history): +def test_ohlcv(mocker, default_conf, ohlcv_history): default_conf["runmode"] = RunMode.DRY_RUN timeframe = default_conf["ticker_interval"] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines[("XRP/BTC", timeframe)] = ticker_history - exchange._klines[("UNITTEST/BTC", timeframe)] = ticker_history + exchange._klines[("XRP/BTC", timeframe)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", timeframe)] = ohlcv_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN - assert ticker_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe)) + assert ohlcv_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe)) assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe), DataFrame) - assert dp.ohlcv("UNITTEST/BTC", timeframe) is not ticker_history - assert dp.ohlcv("UNITTEST/BTC", timeframe, copy=False) is ticker_history + assert dp.ohlcv("UNITTEST/BTC", timeframe) is not ohlcv_history + assert dp.ohlcv("UNITTEST/BTC", timeframe, copy=False) is ohlcv_history assert not dp.ohlcv("UNITTEST/BTC", timeframe).empty assert dp.ohlcv("NONESENSE/AAA", timeframe).empty @@ -37,8 +40,8 @@ def test_ohlcv(mocker, default_conf, ticker_history): assert dp.ohlcv("UNITTEST/BTC", timeframe).empty -def test_historic_ohlcv(mocker, default_conf, ticker_history): - historymock = MagicMock(return_value=ticker_history) +def test_historic_ohlcv(mocker, default_conf, ohlcv_history): + historymock = MagicMock(return_value=ohlcv_history) mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) dp = DataProvider(default_conf, None) @@ -48,24 +51,24 @@ def test_historic_ohlcv(mocker, default_conf, ticker_history): assert historymock.call_args_list[0][1]["timeframe"] == "5m" -def test_get_pair_dataframe(mocker, default_conf, ticker_history): +def test_get_pair_dataframe(mocker, default_conf, ohlcv_history): default_conf["runmode"] = RunMode.DRY_RUN ticker_interval = default_conf["ticker_interval"] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history - exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history + exchange._klines[("XRP/BTC", ticker_interval)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", ticker_interval)] = ohlcv_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN - assert ticker_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)) + assert ohlcv_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)) assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) - assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval) is not ticker_history + assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval) is not ohlcv_history assert not dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval).empty assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty # Test with and without parameter - assert dp.get_pair_dataframe("UNITTEST/BTC", - ticker_interval).equals(dp.get_pair_dataframe("UNITTEST/BTC")) + assert dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval)\ + .equals(dp.get_pair_dataframe("UNITTEST/BTC")) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) @@ -73,7 +76,7 @@ def test_get_pair_dataframe(mocker, default_conf, ticker_history): assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", ticker_interval), DataFrame) assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty - historymock = MagicMock(return_value=ticker_history) + historymock = MagicMock(return_value=ohlcv_history) mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) default_conf["runmode"] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) @@ -82,21 +85,18 @@ def test_get_pair_dataframe(mocker, default_conf, ticker_history): # assert dp.get_pair_dataframe("NONESENSE/AAA", ticker_interval).empty -def test_available_pairs(mocker, default_conf, ticker_history): +def test_available_pairs(mocker, default_conf, ohlcv_history): exchange = get_patched_exchange(mocker, default_conf) ticker_interval = default_conf["ticker_interval"] - exchange._klines[("XRP/BTC", ticker_interval)] = ticker_history - exchange._klines[("UNITTEST/BTC", ticker_interval)] = ticker_history + exchange._klines[("XRP/BTC", ticker_interval)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", ticker_interval)] = ohlcv_history dp = DataProvider(default_conf, exchange) assert len(dp.available_pairs) == 2 - assert dp.available_pairs == [ - ("XRP/BTC", ticker_interval), - ("UNITTEST/BTC", ticker_interval), - ] + assert dp.available_pairs == [("XRP/BTC", ticker_interval), ("UNITTEST/BTC", ticker_interval), ] -def test_refresh(mocker, default_conf, ticker_history): +def test_refresh(mocker, default_conf, ohlcv_history): refresh_mock = MagicMock() mocker.patch("freqtrade.exchange.Exchange.refresh_latest_ohlcv", refresh_mock) @@ -152,3 +152,45 @@ def test_market(mocker, default_conf, markets): res = dp.market('UNITTEST/BTC') assert res is None + + +def test_ticker(mocker, default_conf, tickers): + ticker_mock = MagicMock(return_value=tickers()['ETH/BTC']) + mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock) + exchange = get_patched_exchange(mocker, default_conf) + dp = DataProvider(default_conf, exchange) + res = dp.ticker('ETH/BTC') + assert type(res) is dict + assert 'symbol' in res + assert res['symbol'] == 'ETH/BTC' + + ticker_mock = MagicMock(side_effect=DependencyException('Pair not found')) + mocker.patch("freqtrade.exchange.Exchange.fetch_ticker", ticker_mock) + exchange = get_patched_exchange(mocker, default_conf) + dp = DataProvider(default_conf, exchange) + res = dp.ticker('UNITTEST/BTC') + assert res == {} + + +def test_current_whitelist(mocker, default_conf, tickers): + # patch default conf to volumepairlist + default_conf['pairlists'][0] = {'method': 'VolumePairList', "number_assets": 5} + + mocker.patch.multiple('freqtrade.exchange.Exchange', + exchange_has=MagicMock(return_value=True), + get_tickers=tickers) + exchange = get_patched_exchange(mocker, default_conf) + + pairlist = PairListManager(exchange, default_conf) + dp = DataProvider(default_conf, exchange, pairlist) + + # Simulate volumepairs from exchange. + pairlist.refresh_pairlist() + + assert dp.current_whitelist() == pairlist._whitelist + # The identity of the 2 lists should be identical + assert dp.current_whitelist() is pairlist._whitelist + + with pytest.raises(OperationalException): + dp = DataProvider(default_conf, exchange) + dp.current_whitelist() diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 7b3143db9..6fd4d9569 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -7,24 +7,24 @@ from shutil import copyfile from unittest.mock import MagicMock, PropertyMock import arrow +import pytest from pandas import DataFrame +from pandas.testing import assert_frame_equal from freqtrade.configuration import TimeRange -from freqtrade.data.history import (_download_pair_history, - _download_trades_history, - _load_cached_data_for_updating, - convert_trades_to_ohlcv, get_timerange, - load_data, load_pair_history, - load_tickerdata_file, pair_data_filename, - pair_trades_filename, - refresh_backtest_ohlcv_data, - refresh_backtest_trades_data, - refresh_data, - trim_dataframe, trim_tickerlist, - validate_backtest_data) +from freqtrade.data.converter import ohlcv_to_dataframe +from freqtrade.data.history.history_utils import ( + _download_pair_history, _download_trades_history, + _load_cached_data_for_updating, convert_trades_to_ohlcv, get_timerange, + load_data, load_pair_history, refresh_backtest_ohlcv_data, + refresh_backtest_trades_data, refresh_data, validate_backtest_data) +from freqtrade.data.history.idatahandler import (IDataHandler, get_datahandler, + get_datahandlerclass) +from freqtrade.data.history.jsondatahandler import (JsonDataHandler, + JsonGzDataHandler) from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json -from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.resolvers import StrategyResolver from tests.conftest import (get_patched_exchange, log_has, log_has_re, patch_exchange) @@ -63,7 +63,7 @@ def _clean_test_file(file: Path) -> None: file_swp.rename(file) -def test_load_data_30min_ticker(mocker, caplog, default_conf, testdatadir) -> None: +def test_load_data_30min_timeframe(mocker, caplog, default_conf, testdatadir) -> None: ld = load_pair_history(pair='UNITTEST/BTC', timeframe='30m', datadir=testdatadir) assert isinstance(ld, DataFrame) assert not log_has( @@ -72,7 +72,7 @@ def test_load_data_30min_ticker(mocker, caplog, default_conf, testdatadir) -> No ) -def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> None: +def test_load_data_7min_timeframe(mocker, caplog, default_conf, testdatadir) -> None: ld = load_pair_history(pair='UNITTEST/BTC', timeframe='7m', datadir=testdatadir) assert isinstance(ld, DataFrame) assert ld.empty @@ -82,8 +82,8 @@ def test_load_data_7min_ticker(mocker, caplog, default_conf, testdatadir) -> Non ) -def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history) +def test_load_data_1min_timeframe(ohlcv_history, mocker, caplog, testdatadir) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history) file = testdatadir / 'UNITTEST_BTC-1m.json' _backup_file(file, copy_file=True) load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC']) @@ -96,8 +96,9 @@ def test_load_data_1min_ticker(ticker_history, mocker, caplog, testdatadir) -> N def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: - ltfmock = mocker.patch('freqtrade.data.history.load_tickerdata_file', - MagicMock(return_value=None)) + ltfmock = mocker.patch( + 'freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load', + MagicMock(return_value=DataFrame())) timerange = TimeRange('date', None, 1510639620, 0) load_pair_history(pair='UNITTEST/BTC', timeframe='1m', datadir=testdatadir, timerange=timerange, @@ -109,12 +110,12 @@ def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> assert ltfmock.call_args_list[0][1]['timerange'].startts == timerange.startts - 20 * 60 -def test_load_data_with_new_pair_1min(ticker_history_list, mocker, caplog, +def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, default_conf, testdatadir) -> None: """ - Test load_pair_history() with 1 min ticker + Test load_pair_history() with 1 min timeframe """ - mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list) + mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) file = testdatadir / 'MEME_BTC-1m.json' @@ -143,27 +144,52 @@ def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) -def test_pair_data_filename(): - fn = pair_data_filename(Path('freqtrade/hello/world'), 'ETH/BTC', '5m') +@pytest.mark.parametrize("pair,expected_result", [ + ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json'), + ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json'), + ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json'), + (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json'), + ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json'), + ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json'), +]) +def test_json_pair_data_filename(pair, expected_result): + fn = JsonDataHandler._pair_data_filename(Path('freqtrade/hello/world'), pair, '5m') assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-5m.json') - - -def test_pair_trades_filename(): - fn = pair_trades_filename(Path('freqtrade/hello/world'), 'ETH/BTC') + assert fn == Path(expected_result) + fn = JsonGzDataHandler._pair_data_filename(Path('freqtrade/hello/world'), pair, '5m') assert isinstance(fn, Path) - assert fn == Path('freqtrade/hello/world/ETH_BTC-trades.json.gz') + assert fn == Path(expected_result + '.gz') -def test_load_cached_data_for_updating(mocker) -> None: - datadir = Path(__file__).parent.parent.joinpath('testdata') +@pytest.mark.parametrize("pair,expected_result", [ + ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'), + ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), + ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'), + (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'), + ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-trades.json'), + ("ACC_OLD_BTC", 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'), +]) +def test_json_pair_trades_filename(pair, expected_result): + fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) + assert isinstance(fn, Path) + assert fn == Path(expected_result) + + fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) + assert isinstance(fn, Path) + assert fn == Path(expected_result + '.gz') + + +def test_load_cached_data_for_updating(mocker, testdatadir) -> None: + + data_handler = get_datahandler(testdatadir, 'json') test_data = None - test_filename = datadir.joinpath('UNITTEST_BTC-1m.json') + test_filename = testdatadir.joinpath('UNITTEST_BTC-1m.json') with open(test_filename, "rt") as file: test_data = json.load(file) - # change now time to test 'line' cases + test_data_df = ohlcv_to_dataframe(test_data, '1m', 'UNITTEST/BTC', + fill_missing=False, drop_incomplete=False) # now = last cached item + 1 hour now_ts = test_data[-1][0] / 1000 + 60 * 60 mocker.patch('arrow.utcnow', return_value=arrow.get(now_ts)) @@ -171,77 +197,41 @@ def test_load_cached_data_for_updating(mocker) -> None: # timeframe starts earlier than the cached data # should fully update data timerange = TimeRange('date', None, test_data[0][0] / 1000 - 1, 0) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == [] + data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler) + assert data.empty assert start_ts == test_data[0][0] - 1000 - # same with 'line' timeframe - num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 120 - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', - TimeRange(None, 'line', 0, -num_lines)) - assert data == [] - assert start_ts < test_data[0][0] - 1 - # timeframe starts in the center of the cached data # should return the chached data w/o the last item timerange = TimeRange('date', None, test_data[0][0] / 1000 + 1, 0) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] + data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler) - # same with 'line' timeframe - num_lines = (test_data[-1][0] - test_data[1][0]) / 1000 / 60 + 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] + assert_frame_equal(data, test_data_df.iloc[:-1]) + assert test_data[-2][0] <= start_ts < test_data[-1][0] # timeframe starts after the chached data # should return the chached data w/o the last item - timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 1, 0) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - - # Try loading last 30 lines. - # Not supported by _load_cached_data_for_updating, we always need to get the full data. - num_lines = 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] - - # no timeframe is set - # should return the chached data w/o the last item - num_lines = 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = _load_cached_data_for_updating(datadir, 'UNITTEST/BTC', '1m', timerange) - assert data == test_data[:-1] - assert test_data[-2][0] < start_ts < test_data[-1][0] + timerange = TimeRange('date', None, test_data[-1][0] / 1000 + 100, 0) + data, start_ts = _load_cached_data_for_updating('UNITTEST/BTC', '1m', timerange, data_handler) + assert_frame_equal(data, test_data_df.iloc[:-1]) + assert test_data[-2][0] <= start_ts < test_data[-1][0] # no datafile exist # should return timestamp start time timerange = TimeRange('date', None, now_ts - 10000, 0) - data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) - assert data == [] + data, start_ts = _load_cached_data_for_updating('NONEXIST/BTC', '1m', timerange, data_handler) + assert data.empty assert start_ts == (now_ts - 10000) * 1000 - # same with 'line' timeframe - num_lines = 30 - timerange = TimeRange(None, 'line', 0, -num_lines) - data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', timerange) - assert data == [] - assert start_ts == (now_ts - num_lines * 60) * 1000 - # no datafile exist, no timeframe is set # should return an empty array and None - data, start_ts = _load_cached_data_for_updating(datadir, 'NONEXIST/BTC', '1m', None) - assert data == [] + data, start_ts = _load_cached_data_for_updating('NONEXIST/BTC', '1m', None, data_handler) + assert data.empty assert start_ts is None -def test_download_pair_history(ticker_history_list, mocker, default_conf, testdatadir) -> None: - mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ticker_history_list) +def test_download_pair_history(ohlcv_history_list, mocker, default_conf, testdatadir) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) file1_1 = testdatadir / 'MEME_BTC-1m.json' file1_5 = testdatadir / 'MEME_BTC-5m.json' @@ -293,7 +283,9 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: [1509836520000, 0.00162008, 0.00162008, 0.00162008, 0.00162008, 108.14853839], [1509836580000, 0.00161, 0.00161, 0.00161, 0.00161, 82.390199] ] - json_dump_mock = mocker.patch('freqtrade.misc.file_dump_json', return_value=None) + json_dump_mock = mocker.patch( + 'freqtrade.data.history.jsondatahandler.JsonDataHandler.ohlcv_store', + return_value=None) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=tick) exchange = get_patched_exchange(mocker, default_conf) _download_pair_history(testdatadir, exchange, pair="UNITTEST/BTC", timeframe='1m') @@ -301,7 +293,7 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: assert json_dump_mock.call_count == 2 -def test_download_backtesting_data_exception(ticker_history, mocker, caplog, +def test_download_backtesting_data_exception(ohlcv_history, mocker, caplog, default_conf, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', side_effect=Exception('File Error')) @@ -325,30 +317,19 @@ def test_download_backtesting_data_exception(ticker_history, mocker, caplog, ) -def test_load_tickerdata_file(testdatadir) -> None: - # 7 does not exist in either format. - assert not load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '7m') - # 1 exists only as a .json - tickerdata = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m') - assert _BTC_UNITTEST_LENGTH == len(tickerdata) - # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json - tickerdata = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '8m') - assert _BTC_UNITTEST_LENGTH == len(tickerdata) - - def test_load_partial_missing(testdatadir, caplog) -> None: # Make sure we start fresh - test missing data at start start = arrow.get('2018-01-01T00:00:00') end = arrow.get('2018-01-11T00:00:00') - tickerdata = load_data(testdatadir, '5m', ['UNITTEST/BTC'], startup_candles=20, - timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) + data = load_data(testdatadir, '5m', ['UNITTEST/BTC'], startup_candles=20, + timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) assert log_has( 'Using indicator startup period: 20 ...', caplog ) # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 - assert td != len(tickerdata['UNITTEST/BTC']) - start_real = tickerdata['UNITTEST/BTC'].iloc[0, 0] + assert td != len(data['UNITTEST/BTC']) + start_real = data['UNITTEST/BTC'].iloc[0, 0] assert log_has(f'Missing data at start for pair ' f'UNITTEST/BTC, data starts at {start_real.strftime("%Y-%m-%d %H:%M:%S")}', caplog) @@ -356,13 +337,14 @@ def test_load_partial_missing(testdatadir, caplog) -> None: caplog.clear() start = arrow.get('2018-01-10T00:00:00') end = arrow.get('2018-02-20T00:00:00') - tickerdata = load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], - timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) + data = load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], + timerange=TimeRange('date', 'date', start.timestamp, end.timestamp)) # timedifference in 5 minutes td = ((end - start).total_seconds() // 60 // 5) + 1 - assert td != len(tickerdata['UNITTEST/BTC']) + assert td != len(data['UNITTEST/BTC']) + # Shift endtime with +5 - as last candle is dropped (partial candle) - end_real = arrow.get(tickerdata['UNITTEST/BTC'].iloc[-1, 0]).shift(minutes=5) + end_real = arrow.get(data['UNITTEST/BTC'].iloc[-1, 0]).shift(minutes=5) assert log_has(f'Missing data at end for pair ' f'UNITTEST/BTC, data ends at {end_real.strftime("%Y-%m-%d %H:%M:%S")}', caplog) @@ -370,7 +352,7 @@ def test_load_partial_missing(testdatadir, caplog) -> None: def test_init(default_conf, mocker) -> None: assert {} == load_data( - datadir='', + datadir=Path(''), pairs=[], timeframe=default_conf['ticker_interval'] ) @@ -379,110 +361,18 @@ def test_init(default_conf, mocker) -> None: def test_init_with_refresh(default_conf, mocker) -> None: exchange = get_patched_exchange(mocker, default_conf) refresh_data( - datadir='', + datadir=Path(''), pairs=[], timeframe=default_conf['ticker_interval'], exchange=exchange ) assert {} == load_data( - datadir='', + datadir=Path(''), pairs=[], timeframe=default_conf['ticker_interval'] ) -def test_trim_tickerlist(testdatadir) -> None: - file = testdatadir / 'UNITTEST_BTC-1m.json' - with open(file) as data_file: - ticker_list = json.load(data_file) - ticker_list_len = len(ticker_list) - - # Test the pattern ^(\d{8})-(\d{8})$ - # This pattern extract a window between the dates - timerange = TimeRange('date', 'date', ticker_list[5][0] / 1000, ticker_list[10][0] / 1000 - 1) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 5 - assert ticker_list[0] is not ticker[0] # The first element should be different - assert ticker_list[5] is ticker[0] # The list starts at the index 5 - assert ticker_list[9] is ticker[-1] # The list ends at the index 9 (5 elements) - - # Test the pattern ^-(\d{8})$ - # This pattern extracts elements from the start to the date - timerange = TimeRange(None, 'date', 0, ticker_list[10][0] / 1000 - 1) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == 10 - assert ticker_list[0] is ticker[0] # The start of the list is included - assert ticker_list[9] is ticker[-1] # The element 10 is not included - - # Test the pattern ^(\d{8})-$ - # This pattern extracts elements from the date to now - timerange = TimeRange('date', None, ticker_list[10][0] / 1000 - 1, None) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_len == ticker_list_len - 10 - assert ticker_list[10] is ticker[0] # The first element is element #10 - assert ticker_list[-1] is ticker[-1] # The last element is the same - - # Test a wrong pattern - # This pattern must return the list unchanged - timerange = TimeRange(None, None, None, 5) - ticker = trim_tickerlist(ticker_list, timerange) - ticker_len = len(ticker) - - assert ticker_list_len == ticker_len - - # passing empty list - timerange = TimeRange(None, None, None, 5) - ticker = trim_tickerlist([], timerange) - assert 0 == len(ticker) - assert not ticker - - -def test_trim_dataframe(testdatadir) -> None: - data = load_data( - datadir=testdatadir, - timeframe='1m', - pairs=['UNITTEST/BTC'] - )['UNITTEST/BTC'] - min_date = int(data.iloc[0]['date'].timestamp()) - max_date = int(data.iloc[-1]['date'].timestamp()) - data_modify = data.copy() - - # Remove first 30 minutes (1800 s) - tr = TimeRange('date', None, min_date + 1800, 0) - data_modify = trim_dataframe(data_modify, tr) - assert not data_modify.equals(data) - assert len(data_modify) < len(data) - assert len(data_modify) == len(data) - 30 - assert all(data_modify.iloc[-1] == data.iloc[-1]) - assert all(data_modify.iloc[0] == data.iloc[30]) - - data_modify = data.copy() - # Remove last 30 minutes (1800 s) - tr = TimeRange(None, 'date', 0, max_date - 1800) - data_modify = trim_dataframe(data_modify, tr) - assert not data_modify.equals(data) - assert len(data_modify) < len(data) - assert len(data_modify) == len(data) - 30 - assert all(data_modify.iloc[0] == data.iloc[0]) - assert all(data_modify.iloc[-1] == data.iloc[-31]) - - data_modify = data.copy() - # Remove first 25 and last 30 minutes (1800 s) - tr = TimeRange('date', 'date', min_date + 1500, max_date - 1800) - data_modify = trim_dataframe(data_modify, tr) - assert not data_modify.equals(data) - assert len(data_modify) < len(data) - assert len(data_modify) == len(data) - 55 - # first row matches 25th original row - assert all(data_modify.iloc[0] == data.iloc[25]) - - def test_file_dump_json_tofile(testdatadir) -> None: file = testdatadir / 'test_{id}.json'.format(id=str(uuid.uuid4())) data = {'bar': 'foo'} @@ -509,9 +399,11 @@ def test_file_dump_json_tofile(testdatadir) -> None: def test_get_timerange(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) - strategy = DefaultStrategy(default_conf) - data = strategy.tickerdata_to_dataframe( + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) + + data = strategy.ohlcvdata_to_dataframe( load_data( datadir=testdatadir, timeframe='1m', @@ -525,9 +417,11 @@ def test_get_timerange(default_conf, mocker, testdatadir) -> None: def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) -> None: patch_exchange(mocker) - strategy = DefaultStrategy(default_conf) - data = strategy.tickerdata_to_dataframe( + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) + + data = strategy.ohlcvdata_to_dataframe( load_data( datadir=testdatadir, timeframe='1m', @@ -547,10 +441,12 @@ def test_validate_backtest_data_warn(default_conf, mocker, caplog, testdatadir) def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> None: patch_exchange(mocker) - strategy = DefaultStrategy(default_conf) + + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) timerange = TimeRange('index', 'index', 200, 250) - data = strategy.tickerdata_to_dataframe( + data = strategy.ohlcvdata_to_dataframe( load_data( datadir=testdatadir, timeframe='5m', @@ -567,7 +463,8 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', + MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -588,7 +485,8 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history._download_pair_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', + MagicMock()) ex = get_patched_exchange(mocker, default_conf) mocker.patch( @@ -608,7 +506,8 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, testdatadir): - dl_mock = mocker.patch('freqtrade.data.history._download_trades_history', MagicMock()) + dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_trades_history', + MagicMock()) mocker.patch( 'freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets) ) @@ -638,23 +537,34 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad ght_mock) exchange = get_patched_exchange(mocker, default_conf) file1 = testdatadir / 'ETH_BTC-trades.json.gz' - + data_handler = get_datahandler(testdatadir, data_format='jsongz') _backup_file(file1) assert not file1.is_file() - assert _download_trades_history(datadir=testdatadir, exchange=exchange, + assert _download_trades_history(data_handler=data_handler, exchange=exchange, pair='ETH/BTC') assert log_has("New Amount of trades: 5", caplog) assert file1.is_file() + ght_mock.reset_mock() + since_time = int(trades_history[-3][0] // 1000) + since_time2 = int(trades_history[-1][0] // 1000) + timerange = TimeRange('date', None, since_time, 0) + assert _download_trades_history(data_handler=data_handler, exchange=exchange, + pair='ETH/BTC', timerange=timerange) + + assert ght_mock.call_count == 1 + # Check this in seconds - since we had to convert to seconds above too. + assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time2 - 5 + # clean files freshly downloaded _clean_test_file(file1) mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', MagicMock(side_effect=ValueError)) - assert not _download_trades_history(datadir=testdatadir, exchange=exchange, + assert not _download_trades_history(data_handler=data_handler, exchange=exchange, pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) @@ -686,3 +596,84 @@ def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): _clean_test_file(file1) _clean_test_file(file5) + + +def test_jsondatahandler_ohlcv_get_pairs(testdatadir): + pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m') + # Convert to set to avoid failures due to sorting + assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC', + 'XMR/BTC', 'ZEC/BTC', 'ADA/BTC', 'ETC/BTC', 'NXT/BTC', + 'DASH/BTC', 'XRP/ETH'} + + pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m') + assert set(pairs) == {'UNITTEST/BTC'} + + +def test_jsondatahandler_trades_get_pairs(testdatadir): + pairs = JsonGzDataHandler.trades_get_pairs(testdatadir) + # Convert to set to avoid failures due to sorting + assert set(pairs) == {'XRP/ETH', 'XRP/OLD'} + + +def test_jsondatahandler_ohlcv_purge(mocker, testdatadir): + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + mocker.patch.object(Path, "unlink", MagicMock()) + dh = JsonGzDataHandler(testdatadir) + assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + + +def test_jsondatahandler_trades_load(mocker, testdatadir, caplog): + dh = JsonGzDataHandler(testdatadir) + logmsg = "Old trades format detected - converting" + dh.trades_load('XRP/ETH') + assert not log_has(logmsg, caplog) + + # Test conversation is happening + dh.trades_load('XRP/OLD') + assert log_has(logmsg, caplog) + + +def test_jsondatahandler_trades_purge(mocker, testdatadir): + mocker.patch.object(Path, "exists", MagicMock(return_value=False)) + mocker.patch.object(Path, "unlink", MagicMock()) + dh = JsonGzDataHandler(testdatadir) + assert not dh.trades_purge('UNITTEST/NONEXIST') + + mocker.patch.object(Path, "exists", MagicMock(return_value=True)) + assert dh.trades_purge('UNITTEST/NONEXIST') + + +def test_jsondatahandler_ohlcv_append(testdatadir): + dh = JsonGzDataHandler(testdatadir) + with pytest.raises(NotImplementedError): + dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame()) + + +def test_jsondatahandler_trades_append(testdatadir): + dh = JsonGzDataHandler(testdatadir) + with pytest.raises(NotImplementedError): + dh.trades_append('UNITTEST/ETH', []) + + +def test_gethandlerclass(): + cl = get_datahandlerclass('json') + assert cl == JsonDataHandler + assert issubclass(cl, IDataHandler) + cl = get_datahandlerclass('jsongz') + assert cl == JsonGzDataHandler + assert issubclass(cl, IDataHandler) + assert issubclass(cl, JsonDataHandler) + with pytest.raises(ValueError, match=r"No datahandler for .*"): + get_datahandlerclass('DeadBeef') + + +def test_get_datahandler(testdatadir): + dh = get_datahandler(testdatadir, 'json') + assert type(dh) == JsonDataHandler + dh = get_datahandler(testdatadir, 'jsongz') + assert type(dh) == JsonGzDataHandler + dh1 = get_datahandler(testdatadir, 'jsongz', dh) + assert id(dh1) == id(dh) diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index 3a866c0a8..163ceff4b 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -10,8 +10,8 @@ import numpy as np import pytest from pandas import DataFrame, to_datetime -from freqtrade import OperationalException -from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.exceptions import OperationalException +from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has @@ -26,7 +26,7 @@ from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, # 5) Stoploss and sell are hit. should sell on stoploss #################################################################### -ticker_start_time = arrow.get(2018, 10, 3) +tests_start_time = arrow.get(2018, 10, 3) ticker_interval_in_minute = 60 _ohlc = {'date': 0, 'buy': 1, 'open': 2, 'high': 3, 'low': 4, 'close': 5, 'sell': 6, 'volume': 7} @@ -43,10 +43,10 @@ def _validate_ohlc(buy_ohlc_sell_matrice): def _build_dataframe(buy_ohlc_sell_matrice): _validate_ohlc(buy_ohlc_sell_matrice) - tickers = [] + data = [] for ohlc in buy_ohlc_sell_matrice: - ticker = { - 'date': ticker_start_time.shift( + d = { + 'date': tests_start_time.shift( minutes=( ohlc[0] * ticker_interval_in_minute)).timestamp * @@ -57,9 +57,9 @@ def _build_dataframe(buy_ohlc_sell_matrice): 'low': ohlc[4], 'close': ohlc[5], 'sell': ohlc[6]} - tickers.append(ticker) + data.append(d) - frame = DataFrame(tickers) + frame = DataFrame(data) frame['date'] = to_datetime(frame['date'], unit='ms', utc=True, @@ -69,7 +69,7 @@ def _build_dataframe(buy_ohlc_sell_matrice): def _time_on_candle(number): - return np.datetime64(ticker_start_time.shift( + return np.datetime64(tests_start_time.shift( minutes=(number * ticker_interval_in_minute)).timestamp * 1000, 'ms') @@ -158,13 +158,13 @@ def test_edge_results(edge_conf, mocker, caplog, data) -> None: assert len(trades) == len(data.trades) if not results.empty: - assert round(results["profit_percent"].sum(), 3) == round(data.profit_perc, 3) + assert round(results["profit_ratio"].sum(), 3) == round(data.profit_perc, 3) for c, trade in enumerate(data.trades): res = results.iloc[c] assert res.exit_type == trade.sell_reason - assert arrow.get(res.open_time) == _get_frame_time_from_offset(trade.open_tick) - assert arrow.get(res.close_time) == _get_frame_time_from_offset(trade.close_tick) + assert res.open_time == _get_frame_time_from_offset(trade.open_tick).replace(tzinfo=None) + assert res.close_time == _get_frame_time_from_offset(trade.close_tick).replace(tzinfo=None) def test_adjust(mocker, edge_conf): @@ -262,7 +262,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', NEOBTC = [ [ - ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + tests_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, @@ -274,7 +274,7 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', base = 0.002 LTCBTC = [ [ - ticker_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, + tests_start_time.shift(minutes=(x * ticker_interval_in_minute)).timestamp * 1000, math.sin(x * hz) / 1000 + base, math.sin(x * hz) / 1000 + base + 0.0001, math.sin(x * hz) / 1000 + base - 0.0001, @@ -282,16 +282,18 @@ def mocked_load_data(datadir, pairs=[], timeframe='0m', 123.45 ] for x in range(0, 500)] - pairdata = {'NEO/BTC': parse_ticker_dataframe(NEOBTC, '1h', pair="NEO/BTC", fill_missing=True), - 'LTC/BTC': parse_ticker_dataframe(LTCBTC, '1h', pair="LTC/BTC", fill_missing=True)} + pairdata = {'NEO/BTC': ohlcv_to_dataframe(NEOBTC, '1h', pair="NEO/BTC", + fill_missing=True), + 'LTC/BTC': ohlcv_to_dataframe(LTCBTC, '1h', pair="LTC/BTC", + fill_missing=True)} return pairdata def test_edge_process_downloaded_data(mocker, edge_conf): freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) - mocker.patch('freqtrade.data.history.refresh_data', MagicMock()) - mocker.patch('freqtrade.data.history.load_data', mocked_load_data) + mocker.patch('freqtrade.edge.edge_positioning.refresh_data', MagicMock()) + mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) assert edge.calculate() @@ -302,8 +304,8 @@ def test_edge_process_downloaded_data(mocker, edge_conf): def test_edge_process_no_data(mocker, edge_conf, caplog): freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) - mocker.patch('freqtrade.data.history.refresh_data', MagicMock()) - mocker.patch('freqtrade.data.history.load_data', MagicMock(return_value={})) + mocker.patch('freqtrade.edge.edge_positioning.refresh_data', MagicMock()) + mocker.patch('freqtrade.edge.edge_positioning.load_data', MagicMock(return_value={})) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) assert not edge.calculate() @@ -315,8 +317,8 @@ def test_edge_process_no_data(mocker, edge_conf, caplog): def test_edge_process_no_trades(mocker, edge_conf, caplog): freqtrade = get_patched_freqtradebot(mocker, edge_conf) mocker.patch('freqtrade.exchange.Exchange.get_fee', MagicMock(return_value=0.001)) - mocker.patch('freqtrade.data.history.refresh_data', MagicMock()) - mocker.patch('freqtrade.data.history.load_data', mocked_load_data) + mocker.patch('freqtrade.edge.edge_positioning.refresh_data', MagicMock()) + mocker.patch('freqtrade.edge.edge_positioning.load_data', mocked_load_data) # Return empty mocker.patch('freqtrade.edge.Edge._find_trades_for_stoploss_range', MagicMock(return_value=[])) edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) @@ -333,12 +335,16 @@ def test_edge_init_error(mocker, edge_conf,): get_patched_freqtradebot(mocker, edge_conf) -def test_process_expectancy(mocker, edge_conf): +@pytest.mark.parametrize("fee,risk_reward_ratio,expectancy", [ + (0.0005, 306.5384615384, 101.5128205128), + (0.001, 152.6923076923, 50.2307692308), +]) +def test_process_expectancy(mocker, edge_conf, fee, risk_reward_ratio, expectancy): edge_conf['edge']['min_trade_number'] = 2 freqtrade = get_patched_freqtradebot(mocker, edge_conf) def get_fee(*args, **kwargs): - return 0.001 + return fee freqtrade.exchange.get_fee = get_fee edge = Edge(edge_conf, freqtrade.exchange, freqtrade.strategy) @@ -392,9 +398,9 @@ def test_process_expectancy(mocker, edge_conf): assert 'TEST/BTC' in final assert final['TEST/BTC'].stoploss == -0.9 assert round(final['TEST/BTC'].winrate, 10) == 0.3333333333 - assert round(final['TEST/BTC'].risk_reward_ratio, 10) == 306.5384615384 + assert round(final['TEST/BTC'].risk_reward_ratio, 10) == risk_reward_ratio assert round(final['TEST/BTC'].required_risk_reward, 10) == 2.0 - assert round(final['TEST/BTC'].expectancy, 10) == 101.5128205128 + assert round(final['TEST/BTC'].expectancy, 10) == expectancy # Pop last item so no trade is profitable trades.pop() diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index 7720a7d2e..52faa284b 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -4,12 +4,17 @@ from unittest.mock import MagicMock import ccxt import pytest -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from tests.conftest import get_patched_exchange -def test_stoploss_limit_order(default_conf, mocker): +@pytest.mark.parametrize('limitratio,expected', [ + (None, 220 * 0.99), + (0.99, 220 * 0.99), + (0.98, 220 * 0.98), +]) +def test_stoploss_order_binance(default_conf, mocker, limitratio, expected): api_mock = MagicMock() order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) order_type = 'stop_loss_limit' @@ -20,68 +25,70 @@ def test_stoploss_limit_order(default_conf, mocker): 'foo': 'bar' } }) - default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=190, + order_types={'stoploss_on_exchange_limit_ratio': 1.05}) api_mock.create_order.reset_mock() - - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + order_types = {} if limitratio is None else {'stoploss_on_exchange_limit_ratio': limitratio} + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types=order_types) assert 'id' in order assert 'info' in order assert order['id'] == order_id - assert api_mock.create_order.call_args[0][0] == 'ETH/BTC' - assert api_mock.create_order.call_args[0][1] == order_type - assert api_mock.create_order.call_args[0][2] == 'sell' - assert api_mock.create_order.call_args[0][3] == 1 - assert api_mock.create_order.call_args[0][4] == 200 - assert api_mock.create_order.call_args[0][5] == {'stopPrice': 220} + assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC' + assert api_mock.create_order.call_args_list[0][1]['type'] == order_type + assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' + assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 + # Price should be 1% below stopprice + assert api_mock.create_order.call_args_list[0][1]['price'] == expected + assert api_mock.create_order.call_args_list[0][1]['params'] == {'stopPrice': 220} # test exception handling with pytest.raises(DependencyException): api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) with pytest.raises(InvalidOrderException): api_mock.create_order = MagicMock( side_effect=ccxt.InvalidOrder("binance Order would trigger immediately.")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) with pytest.raises(TemporaryError): api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) with pytest.raises(OperationalException, match=r".*DeadBeef.*"): api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) -def test_stoploss_limit_order_dry_run(default_conf, mocker): +def test_stoploss_order_dry_run_binance(default_conf, mocker): api_mock = MagicMock() order_type = 'stop_loss_limit' default_conf['dry_run'] = True - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock, 'binance') with pytest.raises(OperationalException): - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=190, rate=200) + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=190, + order_types={'stoploss_on_exchange_limit_ratio': 1.05}) api_mock.create_order.reset_mock() - order = exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) assert 'id' in order assert 'info' in order @@ -90,3 +97,17 @@ def test_stoploss_limit_order_dry_run(default_conf, mocker): assert order['type'] == order_type assert order['price'] == 220 assert order['amount'] == 1 + + +def test_stoploss_adjust_binance(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf, id='binance') + order = { + 'type': 'stop_loss_limit', + 'price': 1500, + 'info': {'stopPrice': 1500}, + } + assert exchange.stoploss_adjust(1501, order) + assert not exchange.stoploss_adjust(1499, order) + # Test with invalid order case + order['type'] = 'stop_loss' + assert not exchange.stoploss_adjust(1501, order) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 629f99aa2..7b1e9ddaa 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -11,8 +11,8 @@ import ccxt import pytest from pandas import DataFrame -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError) +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.exchange import Binance, Exchange, Kraken from freqtrade.exchange.common import API_RETRY_COUNT from freqtrade.exchange.exchange import (market_is_active, symbol_is_pair, @@ -73,11 +73,14 @@ def test_init(default_conf, mocker, caplog): def test_init_ccxt_kwargs(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') caplog.set_level(logging.INFO) conf = copy.deepcopy(default_conf) - conf['exchange']['ccxt_async_config'] = {'aiohttp_trust_env': True} + conf['exchange']['ccxt_async_config'] = {'aiohttp_trust_env': True, 'asyncio_loop': True} ex = Exchange(conf) - assert log_has("Applying additional ccxt config: {'aiohttp_trust_env': True}", caplog) + assert log_has( + "Applying additional ccxt config: {'aiohttp_trust_env': True, 'asyncio_loop': True}", + caplog) assert ex._api_async.aiohttp_trust_env assert not ex._api.aiohttp_trust_env @@ -85,6 +88,8 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog): caplog.clear() conf = copy.deepcopy(default_conf) conf['exchange']['ccxt_config'] = {'TestKWARG': 11} + conf['exchange']['ccxt_async_config'] = {'asyncio_loop': True} + ex = Exchange(conf) assert not log_has("Applying additional ccxt config: {'aiohttp_trust_env': True}", caplog) assert not ex._api_async.aiohttp_trust_env @@ -121,22 +126,23 @@ def test_init_exception(default_conf, mocker): def test_exchange_resolver(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=MagicMock())) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - exchange = ExchangeResolver('Bittrex', default_conf).exchange + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + exchange = ExchangeResolver.load_exchange('Bittrex', default_conf) assert isinstance(exchange, Exchange) assert log_has_re(r"No .* specific subclass found. Using the generic class instead.", caplog) caplog.clear() - exchange = ExchangeResolver('kraken', default_conf).exchange + exchange = ExchangeResolver.load_exchange('kraken', default_conf) assert isinstance(exchange, Exchange) assert isinstance(exchange, Kraken) assert not isinstance(exchange, Binance) assert not log_has_re(r"No .* specific subclass found. Using the generic class instead.", caplog) - exchange = ExchangeResolver('binance', default_conf).exchange + exchange = ExchangeResolver.load_exchange('binance', default_conf) assert isinstance(exchange, Exchange) assert isinstance(exchange, Binance) assert not isinstance(exchange, Kraken) @@ -145,7 +151,7 @@ def test_exchange_resolver(default_conf, mocker, caplog): caplog) # Test mapping - exchange = ExchangeResolver('binanceus', default_conf).exchange + exchange = ExchangeResolver.load_exchange('binanceus', default_conf) assert isinstance(exchange, Exchange) assert isinstance(exchange, Binance) assert not isinstance(exchange, Kraken) @@ -173,35 +179,104 @@ def test_validate_order_time_in_force(default_conf, mocker, caplog): ex.validate_order_time_in_force(tif2) -def test_symbol_amount_prec(default_conf, mocker): +@pytest.mark.parametrize("amount,precision_mode,precision,expected", [ + (2.34559, 2, 4, 2.3455), + (2.34559, 2, 5, 2.34559), + (2.34559, 2, 3, 2.345), + (2.9999, 2, 3, 2.999), + (2.9909, 2, 3, 2.990), + # Tests for Tick-size + (2.34559, 4, 0.0001, 2.3455), + (2.34559, 4, 0.00001, 2.34559), + (2.34559, 4, 0.001, 2.345), + (2.9999, 4, 0.001, 2.999), + (2.9909, 4, 0.001, 2.990), + (2.9909, 4, 0.005, 2.990), + (2.9999, 4, 0.005, 2.995), +]) +def test_amount_to_precision(default_conf, mocker, amount, precision_mode, precision, expected): ''' - Test rounds down to 4 Decimal places + Test rounds down ''' - markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'amount': 4}}}) + markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'amount': precision}}}) + + exchange = get_patched_exchange(mocker, default_conf, id="binance") + # digits counting mode + # DECIMAL_PLACES = 2 + # SIGNIFICANT_DIGITS = 3 + # TICK_SIZE = 4 + mocker.patch('freqtrade.exchange.Exchange.precisionMode', + PropertyMock(return_value=precision_mode)) + mocker.patch('freqtrade.exchange.Exchange.markets', markets) + + pair = 'ETH/BTC' + assert exchange.amount_to_precision(pair, amount) == expected + + +@pytest.mark.parametrize("price,precision_mode,precision,expected", [ + (2.34559, 2, 4, 2.3456), + (2.34559, 2, 5, 2.34559), + (2.34559, 2, 3, 2.346), + (2.9999, 2, 3, 3.000), + (2.9909, 2, 3, 2.991), + # Tests for Tick_size + (2.34559, 4, 0.0001, 2.3456), + (2.34559, 4, 0.00001, 2.34559), + (2.34559, 4, 0.001, 2.346), + (2.9999, 4, 0.001, 3.000), + (2.9909, 4, 0.001, 2.991), + (2.9909, 4, 0.005, 2.995), + (2.9973, 4, 0.005, 3.0), + (2.9977, 4, 0.005, 3.0), + (234.43, 4, 0.5, 234.5), + (234.53, 4, 0.5, 235.0), + (0.891534, 4, 0.0001, 0.8916), + +]) +def test_price_to_precision(default_conf, mocker, price, precision_mode, precision, expected): + ''' + Test price to precision + ''' + markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'price': precision}}}) exchange = get_patched_exchange(mocker, default_conf, id="binance") mocker.patch('freqtrade.exchange.Exchange.markets', markets) + # digits counting mode + # DECIMAL_PLACES = 2 + # SIGNIFICANT_DIGITS = 3 + # TICK_SIZE = 4 + mocker.patch('freqtrade.exchange.Exchange.precisionMode', + PropertyMock(return_value=precision_mode)) - amount = 2.34559 pair = 'ETH/BTC' - amount = exchange.symbol_amount_prec(pair, amount) - assert amount == 2.3455 + assert pytest.approx(exchange.price_to_precision(pair, price)) == expected -def test_symbol_price_prec(default_conf, mocker): - ''' - Test rounds up to 4 decimal places - ''' - markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'price': 4}}}) +@pytest.mark.parametrize("price,precision_mode,precision,expected", [ + (2.34559, 2, 4, 0.0001), + (2.34559, 2, 5, 0.00001), + (2.34559, 2, 3, 0.001), + (2.9999, 2, 3, 0.001), + (200.0511, 2, 3, 0.001), + # Tests for Tick_size + (2.34559, 4, 0.0001, 0.0001), + (2.34559, 4, 0.00001, 0.00001), + (2.34559, 4, 0.0025, 0.0025), + (2.9909, 4, 0.0025, 0.0025), + (234.43, 4, 0.5, 0.5), + (234.43, 4, 0.0025, 0.0025), + (234.43, 4, 0.00013, 0.00013), +]) +def test_price_get_one_pip(default_conf, mocker, price, precision_mode, precision, expected): + markets = PropertyMock(return_value={'ETH/BTC': {'precision': {'price': precision}}}) exchange = get_patched_exchange(mocker, default_conf, id="binance") mocker.patch('freqtrade.exchange.Exchange.markets', markets) - - price = 2.34559 + mocker.patch('freqtrade.exchange.Exchange.precisionMode', + PropertyMock(return_value=precision_mode)) pair = 'ETH/BTC' - price = exchange.symbol_price_prec(pair, price) - assert price == 2.3456 + assert pytest.approx(exchange.price_get_one_pip(pair, price)) == expected def test_set_sandbox(default_conf, mocker): @@ -257,9 +332,10 @@ def test__load_markets(default_conf, mocker, caplog): api_mock = MagicMock() api_mock.load_markets = MagicMock(side_effect=ccxt.BaseError("SomeError")) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) assert log_has('Unable to initialize markets. Reason: SomeError', caplog) @@ -315,17 +391,83 @@ def test__reload_markets_exception(default_conf, mocker, caplog): assert log_has_re(r"Could not reload markets.*", caplog) +@pytest.mark.parametrize("stake_currency", ['ETH', 'BTC', 'USDT']) +def test_validate_stake_currency(default_conf, stake_currency, mocker, caplog): + default_conf['stake_currency'] = stake_currency + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/ETH': {'quote': 'ETH'}, 'NEO/USDT': {'quote': 'USDT'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + Exchange(default_conf) + + +def test_validate_stake_currency_error(default_conf, mocker, caplog): + default_conf['stake_currency'] = 'XRP' + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/ETH': {'quote': 'ETH'}, 'NEO/USDT': {'quote': 'USDT'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + with pytest.raises(OperationalException, + match=r'XRP is not available as stake on .*' + 'Available currencies are: BTC, ETH, USDT'): + Exchange(default_conf) + + +def test_get_quote_currencies(default_conf, mocker): + ex = get_patched_exchange(mocker, default_conf) + + assert set(ex.get_quote_currencies()) == set(['USD', 'ETH', 'BTC', 'USDT']) + + +@pytest.mark.parametrize('pair,expected', [ + ('XRP/BTC', 'BTC'), + ('LTC/USD', 'USD'), + ('ETH/USDT', 'USDT'), + ('XLTCUSDT', 'USDT'), + ('XRP/NOCURRENCY', ''), +]) +def test_get_pair_quote_currency(default_conf, mocker, pair, expected): + ex = get_patched_exchange(mocker, default_conf) + assert ex.get_pair_quote_currency(pair) == expected + + +@pytest.mark.parametrize('pair,expected', [ + ('XRP/BTC', 'XRP'), + ('LTC/USD', 'LTC'), + ('ETH/USDT', 'ETH'), + ('XLTCUSDT', 'LTC'), + ('XRP/NOCURRENCY', ''), +]) +def test_get_pair_base_currency(default_conf, mocker, pair, expected): + ex = get_patched_exchange(mocker, default_conf) + assert ex.get_pair_base_currency(pair) == expected + + def test_validate_pairs(default_conf, mocker): # test exchange.validate_pairs directly api_mock = MagicMock() type(api_mock).markets = PropertyMock(return_value={ - 'ETH/BTC': {}, 'LTC/BTC': {}, 'XRP/BTC': {}, 'NEO/BTC': {} + 'ETH/BTC': {'quote': 'BTC'}, + 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, + 'NEO/BTC': {'quote': 'BTC'}, }) id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -335,8 +477,9 @@ def test_validate_pairs_not_available(default_conf, mocker): 'XRP/BTC': {'inactive': True} }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') with pytest.raises(OperationalException, match=r'not available'): Exchange(default_conf) @@ -349,8 +492,9 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): type(api_mock).markets = PropertyMock(return_value={}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') with pytest.raises(OperationalException, match=r'Pair ETH/BTC is not available on Binance'): Exchange(default_conf) @@ -363,21 +507,74 @@ def test_validate_pairs_exception(default_conf, mocker, caplog): def test_validate_pairs_restricted(default_conf, mocker, caplog): api_mock = MagicMock() type(api_mock).markets = PropertyMock(return_value={ - 'ETH/BTC': {}, 'LTC/BTC': {}, 'NEO/BTC': {}, - 'XRP/BTC': {'info': {'IsRestricted': True}} + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC', 'info': {'IsRestricted': True}}, + 'NEO/BTC': {'quote': 'BTC', 'info': 'TestString'}, # info can also be a string ... }) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) - assert log_has(f"Pair XRP/BTC is restricted for some users on this exchange." - f"Please check if you are impacted by this restriction " - f"on the exchange and eventually remove XRP/BTC from your whitelist.", caplog) + assert log_has("Pair XRP/BTC is restricted for some users on this exchange." + "Please check if you are impacted by this restriction " + "on the exchange and eventually remove XRP/BTC from your whitelist.", caplog) -def test_validate_timeframes(default_conf, mocker): - default_conf["ticker_interval"] = "5m" +def test_validate_pairs_stakecompatibility(default_conf, mocker, caplog): + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'BTC'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + Exchange(default_conf) + + +def test_validate_pairs_stakecompatibility_downloaddata(default_conf, mocker, caplog): + api_mock = MagicMock() + default_conf['stake_currency'] = '' + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'BTC'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + Exchange(default_conf) + + +def test_validate_pairs_stakecompatibility_fail(default_conf, mocker, caplog): + default_conf['exchange']['pair_whitelist'].append('HELLO-WORLD') + api_mock = MagicMock() + type(api_mock).markets = PropertyMock(return_value={ + 'ETH/BTC': {'quote': 'BTC'}, 'LTC/BTC': {'quote': 'BTC'}, + 'XRP/BTC': {'quote': 'BTC'}, 'NEO/BTC': {'quote': 'BTC'}, + 'HELLO-WORLD': {'quote': 'USDT'}, + }) + mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + + with pytest.raises(OperationalException, match=r"Stake-currency 'BTC' not compatible with.*"): + Exchange(default_conf) + + +@pytest.mark.parametrize("timeframe", [ + ('5m'), ("1m"), ("15m"), ("1h") +]) +def test_validate_timeframes(default_conf, mocker, timeframe): + default_conf["ticker_interval"] = timeframe api_mock = MagicMock() id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock @@ -389,7 +586,8 @@ def test_validate_timeframes(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -398,7 +596,8 @@ def test_validate_timeframes_failed(default_conf, mocker): api_mock = MagicMock() id_mock = PropertyMock(return_value='test_exchange') type(api_mock).id = id_mock - timeframes = PropertyMock(return_value={'1m': '1m', + timeframes = PropertyMock(return_value={'15s': '15s', + '1m': '1m', '5m': '5m', '15m': '15m', '1h': '1h'}) @@ -408,7 +607,12 @@ def test_validate_timeframes_failed(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) with pytest.raises(OperationalException, - match=r"Invalid ticker interval '3m'. This exchange supports.*"): + match=r"Invalid timeframe '3m'. This exchange supports.*"): + Exchange(default_conf) + default_conf["ticker_interval"] = "15s" + + with pytest.raises(OperationalException, + match=r"Timeframes < 1m are currently not supported by Freqtrade."): Exchange(default_conf) @@ -423,7 +627,8 @@ def test_validate_timeframes_emulated_ohlcv_1(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r'The ccxt library does not provide the list of timeframes ' r'for the exchange ".*" and this exchange ' @@ -444,6 +649,7 @@ def test_validate_timeframes_emulated_ohlcvi_2(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={'timeframes': None})) mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') with pytest.raises(OperationalException, match=r'The ccxt library does not provide the list of timeframes ' r'for the exchange ".*" and this exchange ' @@ -464,7 +670,8 @@ def test_validate_timeframes_not_in_config(default_conf, mocker): mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') Exchange(default_conf) @@ -474,8 +681,9 @@ def test_validate_order_types(default_conf, mocker): type(api_mock).has = PropertyMock(return_value={'createMarketOrder': True}) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') mocker.patch('freqtrade.exchange.Exchange.name', 'Bittrex') default_conf['order_types'] = { 'buy': 'limit', @@ -516,8 +724,9 @@ def test_validate_order_types_not_in_config(default_conf, mocker): api_mock = MagicMock() mocker.patch('freqtrade.exchange.Exchange._init_ccxt', MagicMock(return_value=api_mock)) mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') conf = copy.deepcopy(default_conf) Exchange(conf) @@ -528,9 +737,10 @@ def test_validate_required_startup_candles(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange.name', PropertyMock(return_value='Binance')) mocker.patch('freqtrade.exchange.Exchange._init_ccxt', api_mock) - mocker.patch('freqtrade.exchange.Exchange.validate_timeframes', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange._load_async_markets', MagicMock()) - mocker.patch('freqtrade.exchange.Exchange.validate_pairs', MagicMock()) + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange._load_async_markets') + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') default_conf['startup_candle_count'] = 20 ex = Exchange(default_conf) @@ -595,8 +805,8 @@ def test_create_order(default_conf, mocker, side, ordertype, rate, marketprice, } }) default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) order = exchange.create_order( @@ -636,8 +846,8 @@ def test_buy_prod(default_conf, mocker, exchange_name): } }) default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) order = exchange.buy(pair='ETH/BTC', ordertype=order_type, @@ -710,8 +920,8 @@ def test_buy_considers_time_in_force(default_conf, mocker, exchange_name): } }) default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) order_type = 'limit' @@ -772,8 +982,8 @@ def test_sell_prod(default_conf, mocker, exchange_name): }) default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) @@ -836,8 +1046,8 @@ def test_sell_considers_time_in_force(default_conf, mocker, exchange_name): }) api_mock.options = {} default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) order_type = 'limit' @@ -977,7 +1187,7 @@ def test_get_tickers(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_ticker(default_conf, mocker, exchange_name): +def test_fetch_ticker(default_conf, mocker, exchange_name): api_mock = MagicMock() tick = { 'symbol': 'ETH/BTC', @@ -989,7 +1199,7 @@ def test_get_ticker(default_conf, mocker, exchange_name): api_mock.markets = {'ETH/BTC': {'active': True}} exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) # retrieve original ticker - ticker = exchange.get_ticker(pair='ETH/BTC') + ticker = exchange.fetch_ticker(pair='ETH/BTC') assert ticker['bid'] == 0.00001098 assert ticker['ask'] == 0.00001099 @@ -1006,37 +1216,28 @@ def test_get_ticker(default_conf, mocker, exchange_name): # if not caching the result we should get the same ticker # if not fetching a new result we should get the cached ticker - ticker = exchange.get_ticker(pair='ETH/BTC') + ticker = exchange.fetch_ticker(pair='ETH/BTC') assert api_mock.fetch_ticker.call_count == 1 assert ticker['bid'] == 0.5 assert ticker['ask'] == 1 - assert 'ETH/BTC' in exchange._cached_ticker - assert exchange._cached_ticker['ETH/BTC']['bid'] == 0.5 - assert exchange._cached_ticker['ETH/BTC']['ask'] == 1 - - # Test caching - api_mock.fetch_ticker = MagicMock() - exchange.get_ticker(pair='ETH/BTC', refresh=False) - assert api_mock.fetch_ticker.call_count == 0 - ccxt_exceptionhandlers(mocker, default_conf, api_mock, exchange_name, - "get_ticker", "fetch_ticker", - pair='ETH/BTC', refresh=True) + "fetch_ticker", "fetch_ticker", + pair='ETH/BTC') api_mock.fetch_ticker = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - exchange.get_ticker(pair='ETH/BTC', refresh=True) + exchange.fetch_ticker(pair='ETH/BTC') with pytest.raises(DependencyException, match=r'Pair XRP/ETH not available'): - exchange.get_ticker(pair='XRP/ETH', refresh=True) + exchange.fetch_ticker(pair='XRP/ETH') @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) - tick = [ + ohlcv = [ [ arrow.utcnow().timestamp * 1000, # unix timestamp ms 1, # open @@ -1049,7 +1250,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): pair = 'ETH/BTC' async def mock_candle_hist(pair, timeframe, since_ms): - return pair, timeframe, tick + return pair, timeframe, ohlcv exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls @@ -1057,12 +1258,12 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): ret = exchange.get_historic_ohlcv(pair, "5m", int((arrow.utcnow().timestamp - since) * 1000)) assert exchange._async_get_candle_history.call_count == 2 - # Returns twice the above tick + # Returns twice the above OHLCV data assert len(ret) == 2 def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: - tick = [ + ohlcv = [ [ (arrow.utcnow().timestamp - 1) * 1000, # unix timestamp ms 1, # open @@ -1083,14 +1284,14 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf) - exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')] # empty dicts assert not exchange._klines exchange.refresh_latest_ohlcv(pairs) - assert log_has(f'Refreshing ohlcv data for {len(pairs)} pairs', caplog) + assert log_has(f'Refreshing candle (OHLCV) data for {len(pairs)} pairs', caplog) assert exchange._klines assert exchange._api_async.fetch_ohlcv.call_count == 2 for pair in pairs: @@ -1108,14 +1309,15 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]) assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert log_has(f"Using cached ohlcv data for pair {pairs[0][0]}, timeframe {pairs[0][1]} ...", + assert log_has(f"Using cached candle (OHLCV) data for pair {pairs[0][0]}, " + f"timeframe {pairs[0][1]} ...", caplog) @pytest.mark.asyncio @pytest.mark.parametrize("exchange_name", EXCHANGES) async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_name): - tick = [ + ohlcv = [ [ arrow.utcnow().timestamp * 1000, # unix timestamp ms 1, # open @@ -1129,7 +1331,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) # Monkey-patch async function - exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pair = 'ETH/BTC' res = await exchange._async_get_candle_history(pair, "5m") @@ -1137,9 +1339,9 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ assert len(res) == 3 assert res[0] == pair assert res[1] == "5m" - assert res[2] == tick + assert res[2] == ohlcv assert exchange._api_async.fetch_ohlcv.call_count == 1 - assert not log_has(f"Using cached ohlcv data for {pair} ...", caplog) + assert not log_has(f"Using cached candle (OHLCV) data for {pair} ...", caplog) # exchange = Exchange(default_conf) await async_ccxt_exception(mocker, default_conf, MagicMock(), @@ -1147,14 +1349,15 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ pair='ABCD/BTC', timeframe=default_conf['ticker_interval']) api_mock = MagicMock() - with pytest.raises(OperationalException, match=r'Could not fetch ticker data*'): + with pytest.raises(OperationalException, + match=r'Could not fetch historical candle \(OHLCV\) data.*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", (arrow.utcnow().timestamp - 2000) * 1000) with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching ' - r'historical candlestick data\..*'): + r'historical candle \(OHLCV\) data\..*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) await exchange._async_get_candle_history(pair, "5m", @@ -1164,7 +1367,7 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ @pytest.mark.asyncio async def test__async_get_candle_history_empty(default_conf, mocker, caplog): """ Test empty exchange result """ - tick = [] + ohlcv = [] caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf) @@ -1178,7 +1381,7 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog): assert len(res) == 3 assert res[0] == pair assert res[1] == "5m" - assert res[2] == tick + assert res[2] == ohlcv assert exchange._api_async.fetch_ohlcv.call_count == 1 @@ -1256,8 +1459,8 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na return sorted(data, key=key) # GDAX use-case (real data from GDAX) - # This ticker history is ordered DESC (newest first, oldest last) - tick = [ + # This OHLCV data is ordered DESC (newest first, oldest last) + ohlcv = [ [1527833100000, 0.07666, 0.07671, 0.07666, 0.07668, 16.65244264], [1527832800000, 0.07662, 0.07666, 0.07662, 0.07666, 1.30051526], [1527832500000, 0.07656, 0.07661, 0.07656, 0.07661, 12.034778840000001], @@ -1270,31 +1473,31 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na [1527830400000, 0.07649, 0.07651, 0.07649, 0.07651, 2.5734867] ] exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) - exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) sort_mock = mocker.patch('freqtrade.exchange.exchange.sorted', MagicMock(side_effect=sort_data)) - # Test the ticker history sort + # Test the OHLCV data sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' - ticks = res[2] + res_ohlcv = res[2] assert sort_mock.call_count == 1 - assert ticks[0][0] == 1527830400000 - assert ticks[0][1] == 0.07649 - assert ticks[0][2] == 0.07651 - assert ticks[0][3] == 0.07649 - assert ticks[0][4] == 0.07651 - assert ticks[0][5] == 2.5734867 + assert res_ohlcv[0][0] == 1527830400000 + assert res_ohlcv[0][1] == 0.07649 + assert res_ohlcv[0][2] == 0.07651 + assert res_ohlcv[0][3] == 0.07649 + assert res_ohlcv[0][4] == 0.07651 + assert res_ohlcv[0][5] == 2.5734867 - assert ticks[9][0] == 1527833100000 - assert ticks[9][1] == 0.07666 - assert ticks[9][2] == 0.07671 - assert ticks[9][3] == 0.07666 - assert ticks[9][4] == 0.07668 - assert ticks[9][5] == 16.65244264 + assert res_ohlcv[9][0] == 1527833100000 + assert res_ohlcv[9][1] == 0.07666 + assert res_ohlcv[9][2] == 0.07671 + assert res_ohlcv[9][3] == 0.07666 + assert res_ohlcv[9][4] == 0.07668 + assert res_ohlcv[9][5] == 16.65244264 # Bittrex use-case (real data from Bittrex) - # This ticker history is ordered ASC (oldest first, newest last) - tick = [ + # This OHLCV data is ordered ASC (oldest first, newest last) + ohlcv = [ [1527827700000, 0.07659999, 0.0766, 0.07627, 0.07657998, 1.85216924], [1527828000000, 0.07657995, 0.07657995, 0.0763, 0.0763, 26.04051037], [1527828300000, 0.0763, 0.07659998, 0.0763, 0.0764, 10.36434124], @@ -1306,46 +1509,46 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na [1527830100000, 0.076695, 0.07671, 0.07624171, 0.07671, 1.80689244], [1527830400000, 0.07671, 0.07674399, 0.07629216, 0.07655213, 2.31452783] ] - exchange._api_async.fetch_ohlcv = get_mock_coro(tick) + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) # Reset sort mock sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) - # Test the ticker history sort + # Test the OHLCV data sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['ticker_interval']) assert res[0] == 'ETH/BTC' assert res[1] == default_conf['ticker_interval'] - ticks = res[2] + res_ohlcv = res[2] # Sorted not called again - data is already in order assert sort_mock.call_count == 0 - assert ticks[0][0] == 1527827700000 - assert ticks[0][1] == 0.07659999 - assert ticks[0][2] == 0.0766 - assert ticks[0][3] == 0.07627 - assert ticks[0][4] == 0.07657998 - assert ticks[0][5] == 1.85216924 + assert res_ohlcv[0][0] == 1527827700000 + assert res_ohlcv[0][1] == 0.07659999 + assert res_ohlcv[0][2] == 0.0766 + assert res_ohlcv[0][3] == 0.07627 + assert res_ohlcv[0][4] == 0.07657998 + assert res_ohlcv[0][5] == 1.85216924 - assert ticks[9][0] == 1527830400000 - assert ticks[9][1] == 0.07671 - assert ticks[9][2] == 0.07674399 - assert ticks[9][3] == 0.07629216 - assert ticks[9][4] == 0.07655213 - assert ticks[9][5] == 2.31452783 + assert res_ohlcv[9][0] == 1527830400000 + assert res_ohlcv[9][1] == 0.07671 + assert res_ohlcv[9][2] == 0.07674399 + assert res_ohlcv[9][3] == 0.07629216 + assert res_ohlcv[9][4] == 0.07655213 + assert res_ohlcv[9][5] == 2.31452783 @pytest.mark.asyncio @pytest.mark.parametrize("exchange_name", EXCHANGES) async def test__async_fetch_trades(default_conf, mocker, caplog, exchange_name, - trades_history): + fetch_trades_result): caplog.set_level(logging.DEBUG) exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) # Monkey-patch async function - exchange._api_async.fetch_trades = get_mock_coro(trades_history) + exchange._api_async.fetch_trades = get_mock_coro(fetch_trades_result) pair = 'ETH/BTC' res = await exchange._async_fetch_trades(pair, since=None, params=None) assert type(res) is list - assert isinstance(res[0], dict) - assert isinstance(res[1], dict) + assert isinstance(res[0], list) + assert isinstance(res[1], list) assert exchange._api_async.fetch_trades.call_count == 1 assert exchange._api_async.fetch_trades.call_args[0][0] == pair @@ -1391,7 +1594,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang if 'since' in kwargs: # Return first 3 return trades_history[:-2] - elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3]['id']: + elif kwargs.get('params', {}).get(pagination_arg) == trades_history[-3][1]: # Return 2 return trades_history[-3:-1] else: @@ -1401,8 +1604,8 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist) pair = 'ETH/BTC' - ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]-1) + ret = await exchange._async_get_trade_history_id(pair, since=trades_history[0][0], + until=trades_history[-1][0]-1) assert type(ret) is tuple assert ret[0] == pair assert type(ret[1]) is list @@ -1411,7 +1614,7 @@ async def test__async_get_trade_history_id(default_conf, mocker, caplog, exchang fetch_trades_cal = exchange._async_fetch_trades.call_args_list # first call (using since, not fromId) assert fetch_trades_cal[0][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"] + assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] # 2nd call assert fetch_trades_cal[1][0][0] == pair @@ -1427,7 +1630,7 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha caplog.set_level(logging.DEBUG) async def mock_get_trade_hist(pair, *args, **kwargs): - if kwargs['since'] == trades_history[0]["timestamp"]: + if kwargs['since'] == trades_history[0][0]: return trades_history[:-1] else: return trades_history[-1:] @@ -1437,8 +1640,8 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha # Monkey-patch async function exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist) pair = 'ETH/BTC' - ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]-1) + ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0], + until=trades_history[-1][0]-1) assert type(ret) is tuple assert ret[0] == pair assert type(ret[1]) is list @@ -1447,11 +1650,11 @@ async def test__async_get_trade_history_time(default_conf, mocker, caplog, excha fetch_trades_cal = exchange._async_fetch_trades.call_args_list # first call (using since, not fromId) assert fetch_trades_cal[0][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"] + assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] # 2nd call assert fetch_trades_cal[1][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"] + assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] assert log_has_re(r"Stopping because until was reached.*", caplog) @@ -1463,7 +1666,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, caplog.set_level(logging.DEBUG) async def mock_get_trade_hist(pair, *args, **kwargs): - if kwargs['since'] == trades_history[0]["timestamp"]: + if kwargs['since'] == trades_history[0][0]: return trades_history[:-1] else: return [] @@ -1473,8 +1676,8 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, # Monkey-patch async function exchange._async_fetch_trades = MagicMock(side_effect=mock_get_trade_hist) pair = 'ETH/BTC' - ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]-1) + ret = await exchange._async_get_trade_history_time(pair, since=trades_history[0][0], + until=trades_history[-1][0]-1) assert type(ret) is tuple assert ret[0] == pair assert type(ret[1]) is list @@ -1483,7 +1686,7 @@ async def test__async_get_trade_history_time_empty(default_conf, mocker, caplog, fetch_trades_cal = exchange._async_fetch_trades.call_args_list # first call (using since, not fromId) assert fetch_trades_cal[0][0][0] == pair - assert fetch_trades_cal[0][1]['since'] == trades_history[0]["timestamp"] + assert fetch_trades_cal[0][1]['since'] == trades_history[0][0] @pytest.mark.parametrize("exchange_name", EXCHANGES) @@ -1495,8 +1698,8 @@ def test_get_historic_trades(default_conf, mocker, caplog, exchange_name, trades exchange._async_get_trade_history_id = get_mock_coro((pair, trades_history)) exchange._async_get_trade_history_time = get_mock_coro((pair, trades_history)) - ret = exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]) + ret = exchange.get_historic_trades(pair, since=trades_history[0][0], + until=trades_history[-1][0]) # Depending on the exchange, one or the other method should be called assert sum([exchange._async_get_trade_history_id.call_count, @@ -1517,15 +1720,77 @@ def test_get_historic_trades_notsupported(default_conf, mocker, caplog, exchange with pytest.raises(OperationalException, match="This exchange does not suport downloading Trades."): - exchange.get_historic_trades(pair, since=trades_history[0]["timestamp"], - until=trades_history[-1]["timestamp"]) + exchange.get_historic_trades(pair, since=trades_history[0][0], + until=trades_history[-1][0]) @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_cancel_order_dry_run(default_conf, mocker, exchange_name): default_conf['dry_run'] = True exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) - assert exchange.cancel_order(order_id='123', pair='TKN/BTC') is None + assert exchange.cancel_order(order_id='123', pair='TKN/BTC') == {} + + +@pytest.mark.parametrize("exchange_name", EXCHANGES) +@pytest.mark.parametrize("order,result", [ + ({'status': 'closed', 'filled': 10}, False), + ({'status': 'closed', 'filled': 0.0}, True), + ({'status': 'canceled', 'filled': 0.0}, True), + ({'status': 'canceled', 'filled': 10.0}, False), + ({'status': 'unknown', 'filled': 10.0}, False), + ({'result': 'testest123'}, False), + ]) +def test_check_order_canceled_empty(mocker, default_conf, exchange_name, order, result): + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + assert exchange.check_order_canceled_empty(order) == result + + +@pytest.mark.parametrize("exchange_name", EXCHANGES) +@pytest.mark.parametrize("order,result", [ + ({'status': 'closed', 'amount': 10, 'fee': {}}, True), + ({'status': 'closed', 'amount': 0.0, 'fee': {}}, True), + ({'status': 'canceled', 'amount': 0.0, 'fee': {}}, True), + ({'status': 'canceled', 'amount': 10.0}, False), + ({'amount': 10.0, 'fee': {}}, False), + ({'result': 'testest123'}, False), + ('hello_world', False), +]) +def test_is_cancel_order_result_suitable(mocker, default_conf, exchange_name, order, result): + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + assert exchange.is_cancel_order_result_suitable(order) == result + + +@pytest.mark.parametrize("exchange_name", EXCHANGES) +@pytest.mark.parametrize("corder,call_corder,call_forder", [ + ({'status': 'closed', 'amount': 10, 'fee': {}}, 1, 0), + ({'amount': 10, 'fee': {}}, 1, 1), +]) +def test_cancel_order_with_result(default_conf, mocker, exchange_name, corder, + call_corder, call_forder): + default_conf['dry_run'] = False + api_mock = MagicMock() + api_mock.cancel_order = MagicMock(return_value=corder) + api_mock.fetch_order = MagicMock(return_value={}) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + res = exchange.cancel_order_with_result('1234', 'ETH/BTC', 1234) + assert isinstance(res, dict) + assert api_mock.cancel_order.call_count == call_corder + assert api_mock.fetch_order.call_count == call_forder + + +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test_cancel_order_with_result_error(default_conf, mocker, exchange_name, caplog): + default_conf['dry_run'] = False + api_mock = MagicMock() + api_mock.cancel_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order")) + api_mock.fetch_order = MagicMock(side_effect=ccxt.InvalidOrder("Did not find order")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + + res = exchange.cancel_order_with_result('1234', 'ETH/BTC', 1541) + assert isinstance(res, dict) + assert log_has("Could not cancel order 1234.", caplog) + assert log_has("Could not fetch cancelled order 1234.", caplog) + assert res['amount'] == 1541 # Ensure that if not dry_run, we should call API @@ -1653,10 +1918,13 @@ def test_get_fee(default_conf, mocker, exchange_name): 'get_fee', 'calculate_fee', symbol="ETH/BTC") -def test_stoploss_limit_order_unsupported_exchange(default_conf, mocker): +def test_stoploss_order_unsupported_exchange(default_conf, mocker): exchange = get_patched_exchange(mocker, default_conf, 'bittrex') - with pytest.raises(OperationalException, match=r"stoploss_limit is not implemented .*"): - exchange.stoploss_limit(pair='ETH/BTC', amount=1, stop_price=220, rate=200) + with pytest.raises(OperationalException, match=r"stoploss is not implemented .*"): + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + with pytest.raises(OperationalException, match=r"stoploss is not implemented .*"): + exchange.stoploss_adjust(1, {}) def test_merge_ft_has_dict(default_conf, mocker): @@ -1664,7 +1932,9 @@ def test_merge_ft_has_dict(default_conf, mocker): _init_ccxt=MagicMock(return_value=MagicMock()), _load_async_markets=MagicMock(), validate_pairs=MagicMock(), - validate_timeframes=MagicMock()) + validate_timeframes=MagicMock(), + validate_stakecurrency=MagicMock() + ) ex = Exchange(default_conf) assert ex._ft_has == Exchange._ft_has_default @@ -1714,6 +1984,7 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): # 'ETH/BTC': 'active': True # 'ETH/USDT': 'active': True # 'LTC/BTC': 'active': False + # 'LTC/ETH': 'active': True # 'LTC/USD': 'active': True # 'LTC/USDT': 'active': True # 'NEO/BTC': 'active': False @@ -1722,26 +1993,26 @@ def test_get_valid_pair_combination(default_conf, mocker, markets): # 'XRP/BTC': 'active': False # all markets ([], [], False, False, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', + ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # active markets ([], [], False, True, - ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'NEO/BTC', 'TKN/BTC', 'XLTCUSDT', 'XRP/BTC']), # all pairs ([], [], True, False, - ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', + ['BLK/BTC', 'BTT/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), # active pairs ([], [], True, True, - ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'NEO/BTC', + ['BLK/BTC', 'ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'NEO/BTC', 'TKN/BTC', 'XRP/BTC']), # all markets, base=ETH, LTC (['ETH', 'LTC'], [], False, False, - ['ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), + ['ETH/BTC', 'ETH/USDT', 'LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), # all markets, base=LTC (['LTC'], [], False, False, - ['LTC/BTC', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), + ['LTC/BTC', 'LTC/ETH', 'LTC/USD', 'LTC/USDT', 'XLTCUSDT']), # all markets, quote=USDT ([], ['USDT'], False, False, ['ETH/USDT', 'LTC/USDT', 'XLTCUSDT']), @@ -1874,3 +2145,58 @@ def test_symbol_is_pair(market_symbol, base_currency, quote_currency, expected_r ]) def test_market_is_active(market, expected_result) -> None: assert market_is_active(market) == expected_result + + +@pytest.mark.parametrize("order,expected", [ + ([{'fee'}], False), + ({'fee': None}, False), + ({'fee': {'currency': 'ETH/BTC'}}, False), + ({'fee': {'currency': 'ETH/BTC', 'cost': None}}, False), + ({'fee': {'currency': 'ETH/BTC', 'cost': 0.01}}, True), +]) +def test_order_has_fee(order, expected) -> None: + assert Exchange.order_has_fee(order) == expected + + +@pytest.mark.parametrize("order,expected", [ + ({'symbol': 'ETH/BTC', 'fee': {'currency': 'ETH', 'cost': 0.43}}, + (0.43, 'ETH', 0.01)), + ({'symbol': 'ETH/USDT', 'fee': {'currency': 'USDT', 'cost': 0.01}}, + (0.01, 'USDT', 0.01)), + ({'symbol': 'BTC/USDT', 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}}, + (0.34, 'USDT', 0.01)), +]) +def test_extract_cost_curr_rate(mocker, default_conf, order, expected) -> None: + mocker.patch('freqtrade.exchange.Exchange.calculate_fee_rate', MagicMock(return_value=0.01)) + ex = get_patched_exchange(mocker, default_conf) + assert ex.extract_cost_curr_rate(order) == expected + + +@pytest.mark.parametrize("order,expected", [ + # Using base-currency + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.1), + ({'symbol': 'ETH/BTC', 'amount': 0.05, 'cost': 0.05, + 'fee': {'currency': 'ETH', 'cost': 0.004, 'rate': None}}, 0.08), + # Using quote currency + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'BTC', 'cost': 0.005}}, 0.1), + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'BTC', 'cost': 0.002, 'rate': None}}, 0.04), + # Using foreign currency + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'NEO', 'cost': 0.0012}}, 0.001944), + ({'symbol': 'ETH/BTC', 'amount': 2.21, 'cost': 0.02992561, + 'fee': {'currency': 'NEO', 'cost': 0.00027452}}, 0.00074305), + # TODO: More tests here! + # Rate included in return - return as is + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.01}}, 0.01), + ({'symbol': 'ETH/BTC', 'amount': 0.04, 'cost': 0.05, + 'fee': {'currency': 'USDT', 'cost': 0.34, 'rate': 0.005}}, 0.005), +]) +def test_calculate_fee_rate(mocker, default_conf, order, expected) -> None: + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'last': 0.081}) + + ex = get_patched_exchange(mocker, default_conf) + assert ex.calculate_fee_rate(order) == expected diff --git a/tests/exchange/test_kraken.py b/tests/exchange/test_kraken.py index 3ad62d85a..d63dd66cc 100644 --- a/tests/exchange/test_kraken.py +++ b/tests/exchange/test_kraken.py @@ -3,6 +3,11 @@ from random import randint from unittest.mock import MagicMock +import ccxt +import pytest + +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from tests.conftest import get_patched_exchange from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -21,8 +26,8 @@ def test_buy_kraken_trading_agreement(default_conf, mocker): }) default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") order = exchange.buy(pair='ETH/BTC', ordertype=order_type, @@ -53,8 +58,8 @@ def test_sell_kraken_trading_agreement(default_conf, mocker): }) default_conf['dry_run'] = False - mocker.patch('freqtrade.exchange.Exchange.symbol_amount_prec', lambda s, x, y: y) - mocker.patch('freqtrade.exchange.Exchange.symbol_price_prec', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) exchange = get_patched_exchange(mocker, default_conf, api_mock, id="kraken") order = exchange.sell(pair='ETH/BTC', ordertype=order_type, amount=1, rate=200) @@ -149,3 +154,98 @@ def test_get_balances_prod(default_conf, mocker): assert balances['4ST']['used'] == 0.0 ccxt_exceptionhandlers(mocker, default_conf, api_mock, "kraken", "get_balances", "fetch_balance") + + +def test_stoploss_order_kraken(default_conf, mocker): + api_mock = MagicMock() + order_id = 'test_prod_buy_{}'.format(randint(0, 10 ** 6)) + order_type = 'stop-loss' + + api_mock.create_order = MagicMock(return_value={ + 'id': order_id, + 'info': { + 'foo': 'bar' + } + }) + + default_conf['dry_run'] = False + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + + # stoploss_on_exchange_limit_ratio is irrelevant for kraken market orders + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=190, + order_types={'stoploss_on_exchange_limit_ratio': 1.05}) + assert api_mock.create_order.call_count == 1 + + api_mock.create_order.reset_mock() + + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + assert 'id' in order + assert 'info' in order + assert order['id'] == order_id + assert api_mock.create_order.call_args_list[0][1]['symbol'] == 'ETH/BTC' + assert api_mock.create_order.call_args_list[0][1]['type'] == order_type + assert api_mock.create_order.call_args_list[0][1]['side'] == 'sell' + assert api_mock.create_order.call_args_list[0][1]['amount'] == 1 + assert api_mock.create_order.call_args_list[0][1]['price'] == 220 + assert api_mock.create_order.call_args_list[0][1]['params'] == {'trading_agreement': 'agree'} + + # test exception handling + with pytest.raises(DependencyException): + api_mock.create_order = MagicMock(side_effect=ccxt.InsufficientFunds("0 balance")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + with pytest.raises(InvalidOrderException): + api_mock.create_order = MagicMock( + side_effect=ccxt.InvalidOrder("kraken Order would trigger immediately.")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + with pytest.raises(TemporaryError): + api_mock.create_order = MagicMock(side_effect=ccxt.NetworkError("No connection")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + with pytest.raises(OperationalException, match=r".*DeadBeef.*"): + api_mock.create_order = MagicMock(side_effect=ccxt.BaseError("DeadBeef")) + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + +def test_stoploss_order_dry_run_kraken(default_conf, mocker): + api_mock = MagicMock() + order_type = 'stop-loss' + default_conf['dry_run'] = True + mocker.patch('freqtrade.exchange.Exchange.amount_to_precision', lambda s, x, y: y) + mocker.patch('freqtrade.exchange.Exchange.price_to_precision', lambda s, x, y: y) + + exchange = get_patched_exchange(mocker, default_conf, api_mock, 'kraken') + + api_mock.create_order.reset_mock() + + order = exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) + + assert 'id' in order + assert 'info' in order + assert 'type' in order + + assert order['type'] == order_type + assert order['price'] == 220 + assert order['amount'] == 1 + + +def test_stoploss_adjust_kraken(mocker, default_conf): + exchange = get_patched_exchange(mocker, default_conf, id='kraken') + order = { + 'type': 'stop-loss', + 'price': 1500, + } + assert exchange.stoploss_adjust(1501, order) + assert not exchange.stoploss_adjust(1499, order) + # Test with invalid order case ... + order['type'] = 'stop_loss_limit' + assert not exchange.stoploss_adjust(1501, order) diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index 8756143a0..8bc66f02c 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -1,4 +1,4 @@ -from typing import Dict, List, NamedTuple +from typing import Dict, List, NamedTuple, Optional import arrow from pandas import DataFrame @@ -6,7 +6,7 @@ from pandas import DataFrame from freqtrade.exchange import timeframe_to_minutes from freqtrade.strategy.interface import SellType -ticker_start_time = arrow.get(2018, 10, 3) +tests_start_time = arrow.get(2018, 10, 3) tests_timeframe = '1h' @@ -23,27 +23,27 @@ class BTContainer(NamedTuple): """ Minimal BacktestContainer defining Backtest inputs and results. """ - data: List[float] + data: List[List[float]] stop_loss: float roi: Dict[str, float] trades: List[BTrade] profit_perc: float trailing_stop: bool = False trailing_only_offset_is_reached: bool = False - trailing_stop_positive: float = None + trailing_stop_positive: Optional[float] = None trailing_stop_positive_offset: float = 0.0 use_sell_signal: bool = False def _get_frame_time_from_offset(offset): - return ticker_start_time.shift(minutes=(offset * timeframe_to_minutes(tests_timeframe)) - ).datetime + minutes = offset * timeframe_to_minutes(tests_timeframe) + return tests_start_time.shift(minutes=minutes).datetime -def _build_backtest_dataframe(ticker_with_signals): +def _build_backtest_dataframe(data): columns = ['date', 'open', 'high', 'low', 'close', 'volume', 'buy', 'sell'] - frame = DataFrame.from_records(ticker_with_signals, columns=columns) + frame = DataFrame.from_records(data, columns=columns) frame['date'] = frame['date'].apply(_get_frame_time_from_offset) # Ensure floats are in place for column in ['open', 'high', 'low', 'close', 'volume']: diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index 47cb9f353..e7bc76c1d 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -364,7 +364,7 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: default_conf["trailing_stop"] = data.trailing_stop default_conf["trailing_only_offset_is_reached"] = data.trailing_only_offset_is_reached # Only add this to configuration If it's necessary - if data.trailing_stop_positive: + if data.trailing_stop_positive is not None: default_conf["trailing_stop_positive"] = data.trailing_stop_positive default_conf["trailing_stop_positive_offset"] = data.trailing_stop_positive_offset default_conf["ask_strategy"] = {"use_sell_signal": data.use_sell_signal} @@ -382,13 +382,11 @@ def test_backtest_results(default_conf, fee, mocker, caplog, data) -> None: data_processed = {pair: frame.copy()} min_date, max_date = get_timerange({pair: frame}) results = backtesting.backtest( - { - 'stake_amount': default_conf['stake_amount'], - 'processed': data_processed, - 'max_open_trades': 10, - 'start_date': min_date, - 'end_date': max_date, - } + processed=data_processed, + stake_amount=default_conf['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=10, ) assert len(results) == len(data.trades) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 0ea35e41f..019914720 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -1,31 +1,31 @@ # pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument -import math import random from pathlib import Path -from unittest.mock import MagicMock +from unittest.mock import MagicMock, PropertyMock import numpy as np import pandas as pd import pytest from arrow import Arrow -from freqtrade import DependencyException, OperationalException, constants +from freqtrade import constants +from freqtrade.commands.optimize_commands import (setup_optimize_configuration, + start_backtesting) from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import evaluate_result_multi -from freqtrade.data.converter import parse_ticker_dataframe +from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange -from freqtrade.optimize import setup_configuration, start_backtesting +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.optimize.backtesting import Backtesting +from freqtrade.resolvers import StrategyResolver from freqtrade.state import RunMode -from freqtrade.strategy.default_strategy import DefaultStrategy from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) - ORDER_TYPES = [ { 'buy': 'limit', @@ -50,47 +50,33 @@ def trim_dictlist(dict_list, num): def load_data_test(what, testdatadir): timerange = TimeRange.parse_timerange('1510694220-1510700340') - pair = history.load_tickerdata_file(testdatadir, timeframe='1m', - pair='UNITTEST/BTC', timerange=timerange) - datalen = len(pair) + data = history.load_pair_history(pair='UNITTEST/BTC', datadir=testdatadir, + timeframe='1m', timerange=timerange, + drop_incomplete=False, + fill_up_missing=False) base = 0.001 if what == 'raise': - data = [ - [ - pair[x][0], # Keep old dates - x * base, # But replace O,H,L,C - x * base + 0.0001, - x * base - 0.0001, - x * base, - pair[x][5], # Keep old volume - ] for x in range(0, datalen) - ] + data.loc[:, 'open'] = data.index * base + data.loc[:, 'high'] = data.index * base + 0.0001 + data.loc[:, 'low'] = data.index * base - 0.0001 + data.loc[:, 'close'] = data.index * base + if what == 'lower': - data = [ - [ - pair[x][0], # Keep old dates - 1 - x * base, # But replace O,H,L,C - 1 - x * base + 0.0001, - 1 - x * base - 0.0001, - 1 - x * base, - pair[x][5] # Keep old volume - ] for x in range(0, datalen) - ] + data.loc[:, 'open'] = 1 - data.index * base + data.loc[:, 'high'] = 1 - data.index * base + 0.0001 + data.loc[:, 'low'] = 1 - data.index * base - 0.0001 + data.loc[:, 'close'] = 1 - data.index * base + if what == 'sine': hz = 0.1 # frequency - data = [ - [ - pair[x][0], # Keep old dates - math.sin(x * hz) / 1000 + base, # But replace O,H,L,C - math.sin(x * hz) / 1000 + base + 0.0001, - math.sin(x * hz) / 1000 + base - 0.0001, - math.sin(x * hz) / 1000 + base, - pair[x][5] # Keep old volume - ] for x in range(0, datalen) - ] - return {'UNITTEST/BTC': parse_ticker_dataframe(data, '1m', pair="UNITTEST/BTC", - fill_missing=True)} + data.loc[:, 'open'] = np.sin(data.index * hz) / 1000 + base + data.loc[:, 'high'] = np.sin(data.index * hz) / 1000 + base + 0.0001 + data.loc[:, 'low'] = np.sin(data.index * hz) / 1000 + base - 0.0001 + data.loc[:, 'close'] = np.sin(data.index * hz) / 1000 + base + + return {'UNITTEST/BTC': clean_ohlcv_dataframe(data, timeframe='1m', pair='UNITTEST/BTC', + fill_missing=True)} def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: @@ -99,54 +85,36 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: backtesting = Backtesting(config) data = load_data_test(contour, testdatadir) - processed = backtesting.strategy.tickerdata_to_dataframe(data) + processed = backtesting.strategy.ohlcvdata_to_dataframe(data) min_date, max_date = get_timerange(processed) assert isinstance(processed, dict) results = backtesting.backtest( - { - 'stake_amount': config['stake_amount'], - 'processed': processed, - 'max_open_trades': 1, - 'position_stacking': False, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=config['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, ) # results :: assert len(results) == num_results -def mocked_load_data(datadir, pairs=[], timeframe='0m', - timerange=None, *args, **kwargs): - tickerdata = history.load_tickerdata_file(datadir, 'UNITTEST/BTC', '1m', timerange=timerange) - pairdata = {'UNITTEST/BTC': parse_ticker_dataframe(tickerdata, '1m', pair="UNITTEST/BTC", - fill_missing=True)} - return pairdata - - -# use for mock ccxt.fetch_ohlvc' -def _load_pair_as_ticks(pair, tickfreq): - ticks = history.load_tickerdata_file(None, timeframe=tickfreq, pair=pair) - ticks = ticks[-201:] - return ticks - - # FIX: fixturize this? -def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC', record=None): +def _make_backtest_conf(mocker, datadir, conf=None, pair='UNITTEST/BTC'): data = history.load_data(datadir=datadir, timeframe='1m', pairs=[pair]) data = trim_dictlist(data, -201) patch_exchange(mocker) backtesting = Backtesting(conf) - processed = backtesting.strategy.tickerdata_to_dataframe(data) + processed = backtesting.strategy.ohlcvdata_to_dataframe(data) min_date, max_date = get_timerange(processed) return { - 'stake_amount': conf['stake_amount'], 'processed': processed, - 'max_open_trades': 10, - 'position_stacking': False, - 'record': record, + 'stake_amount': conf['stake_amount'], 'start_date': min_date, 'end_date': max_date, + 'max_open_trades': 10, + 'position_stacking': False, } @@ -180,7 +148,7 @@ def _trend_alternate(dataframe=None, metadata=None): # Unit tests -def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) args = [ @@ -189,7 +157,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> '--strategy', 'DefaultStrategy', ] - config = setup_configuration(get_args(args), RunMode.BACKTEST) + config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -230,7 +198,7 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--fee', '0', ] - config = setup_configuration(get_args(args), RunMode.BACKTEST) + config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -257,13 +225,14 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert 'export' in config assert log_has('Parameter --export detected: {} ...'.format(config['export']), caplog) assert 'exportfilename' in config + assert isinstance(config['exportfilename'], Path) assert log_has('Storing backtest results to {} ...'.format(config['exportfilename']), caplog) assert 'fee' in config assert log_has('Parameter --fee detected, setting fee to: {} ...'.format(config['fee']), caplog) -def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT patched_configuration_load_config_file(mocker, default_conf) @@ -274,8 +243,8 @@ def test_setup_configuration_unlimited_stake_amount(mocker, default_conf, caplog '--strategy', 'DefaultStrategy', ] - with pytest.raises(DependencyException, match=r'.*stake amount.*'): - setup_configuration(get_args(args), RunMode.BACKTEST) + with pytest.raises(DependencyException, match=r'.`stake_amount`.*'): + setup_optimize_configuration(get_args(args), RunMode.BACKTEST) def test_start(mocker, fee, default_conf, caplog) -> None: @@ -290,8 +259,8 @@ def test_start(mocker, fee, default_conf, caplog) -> None: '--config', 'config.json', '--strategy', 'DefaultStrategy', ] - args = get_args(args) - start_backtesting(args) + pargs = get_args(args) + start_backtesting(pargs) assert log_has('Starting freqtrade in Backtesting mode', caplog) assert start_mock.call_count == 1 @@ -308,7 +277,7 @@ def test_backtesting_init(mocker, default_conf, order_types) -> None: backtesting = Backtesting(default_conf) assert backtesting.config == default_conf assert backtesting.timeframe == '5m' - assert callable(backtesting.strategy.tickerdata_to_dataframe) + assert callable(backtesting.strategy.ohlcvdata_to_dataframe) assert callable(backtesting.strategy.advise_buy) assert callable(backtesting.strategy.advise_sell) assert isinstance(backtesting.strategy.dp, DataProvider) @@ -330,7 +299,7 @@ def test_backtesting_init_no_ticker_interval(mocker, default_conf, caplog) -> No "or as cli argument `--ticker-interval 5m`", caplog) -def test_tickerdata_with_fee(default_conf, mocker, testdatadir) -> None: +def test_data_with_fee(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) default_conf['fee'] = 0.1234 @@ -340,138 +309,34 @@ def test_tickerdata_with_fee(default_conf, mocker, testdatadir) -> None: assert fee_mock.call_count == 0 -def test_tickerdata_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: +def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: patch_exchange(mocker) - # timerange = TimeRange(None, 'line', 0, -100) timerange = TimeRange.parse_timerange('1510694220-1510700340') - tick = history.load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", - fill_missing=True)} - + data = history.load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, + fill_up_missing=True) backtesting = Backtesting(default_conf) - data = backtesting.strategy.tickerdata_to_dataframe(tickerlist) - assert len(data['UNITTEST/BTC']) == 102 + processed = backtesting.strategy.ohlcvdata_to_dataframe(data) + assert len(processed['UNITTEST/BTC']) == 102 # Load strategy to compare the result between Backtesting function and strategy are the same - strategy = DefaultStrategy(default_conf) - data2 = strategy.tickerdata_to_dataframe(tickerlist) - assert data['UNITTEST/BTC'].equals(data2['UNITTEST/BTC']) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) - -def test_generate_text_table(default_conf, mocker): - patch_exchange(mocker) - default_conf['max_open_trades'] = 2 - backtesting = Backtesting(default_conf) - - results = pd.DataFrame( - { - 'pair': ['ETH/BTC', 'ETH/BTC'], - 'profit_percent': [0.1, 0.2], - 'profit_abs': [0.2, 0.4], - 'trade_duration': [10, 30], - 'profit': [2, 0], - 'loss': [0, 0] - } - ) - - result_str = ( - '| pair | buy count | avg profit % | cum profit % | ' - 'tot profit BTC | tot profit % | avg duration | profit | loss |\n' - '|:--------|------------:|---------------:|---------------:|' - '-----------------:|---------------:|:---------------|---------:|-------:|\n' - '| ETH/BTC | 2 | 15.00 | 30.00 | ' - '0.60000000 | 15.00 | 0:20:00 | 2 | 0 |\n' - '| TOTAL | 2 | 15.00 | 30.00 | ' - '0.60000000 | 15.00 | 0:20:00 | 2 | 0 |' - ) - assert backtesting._generate_text_table(data={'ETH/BTC': {}}, results=results) == result_str - - -def test_generate_text_table_sell_reason(default_conf, mocker): - patch_exchange(mocker) - backtesting = Backtesting(default_conf) - - results = pd.DataFrame( - { - 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], - 'profit_percent': [0.1, 0.2, 0.3], - 'profit_abs': [0.2, 0.4, 0.5], - 'trade_duration': [10, 30, 10], - 'profit': [2, 0, 0], - 'loss': [0, 0, 1], - 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] - } - ) - - result_str = ( - '| Sell Reason | Count |\n' - '|:--------------|--------:|\n' - '| roi | 2 |\n' - '| stop_loss | 1 |' - ) - assert backtesting._generate_text_table_sell_reason( - data={'ETH/BTC': {}}, results=results) == result_str - - -def test_generate_text_table_strategyn(default_conf, mocker): - """ - Test Backtesting.generate_text_table_sell_reason() method - """ - patch_exchange(mocker) - default_conf['max_open_trades'] = 2 - backtesting = Backtesting(default_conf) - results = {} - results['ETH/BTC'] = pd.DataFrame( - { - 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], - 'profit_percent': [0.1, 0.2, 0.3], - 'profit_abs': [0.2, 0.4, 0.5], - 'trade_duration': [10, 30, 10], - 'profit': [2, 0, 0], - 'loss': [0, 0, 1], - 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] - } - ) - results['LTC/BTC'] = pd.DataFrame( - { - 'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'], - 'profit_percent': [0.4, 0.2, 0.3], - 'profit_abs': [0.4, 0.4, 0.5], - 'trade_duration': [15, 30, 15], - 'profit': [4, 1, 0], - 'loss': [0, 0, 1], - 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] - } - ) - - result_str = ( - '| Strategy | buy count | avg profit % | cum profit % ' - '| tot profit BTC | tot profit % | avg duration | profit | loss |\n' - '|:-----------|------------:|---------------:|---------------:' - '|-----------------:|---------------:|:---------------|---------:|-------:|\n' - '| ETH/BTC | 3 | 20.00 | 60.00 ' - '| 1.10000000 | 30.00 | 0:17:00 | 3 | 0 |\n' - '| LTC/BTC | 3 | 30.00 | 90.00 ' - '| 1.30000000 | 45.00 | 0:20:00 | 3 | 0 |' - ) - assert backtesting._generate_text_table_strategy(all_results=results) == result_str + processed2 = strategy.ohlcvdata_to_dataframe(data) + assert processed['UNITTEST/BTC'].equals(processed2['UNITTEST/BTC']) def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.data.history.load_data', mocked_load_data) mocker.patch('freqtrade.data.history.get_timerange', get_timerange) - mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.optimize.backtesting.Backtesting', - backtest=MagicMock(), - _generate_text_table=MagicMock(return_value='1'), - ) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') + mocker.patch('freqtrade.optimize.backtesting.show_backtest_results') + mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) - default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] default_conf['ticker_interval'] = '1m' default_conf['datadir'] = testdatadir default_conf['export'] = None @@ -494,17 +359,14 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) - mocker.patch('freqtrade.data.history.load_pair_history', MagicMock(return_value=pd.DataFrame())) + mocker.patch('freqtrade.data.history.history_utils.load_pair_history', + MagicMock(return_value=pd.DataFrame())) mocker.patch('freqtrade.data.history.get_timerange', get_timerange) - mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', MagicMock()) patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.optimize.backtesting.Backtesting', - backtest=MagicMock(), - _generate_text_table=MagicMock(return_value='1'), - ) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') + mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) - default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] default_conf['ticker_interval'] = "1m" default_conf['datadir'] = testdatadir default_conf['export'] = None @@ -515,6 +377,29 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> backtesting.start() +def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> None: + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + mocker.patch('freqtrade.data.history.history_utils.load_pair_history', + MagicMock(return_value=pd.DataFrame())) + mocker.patch('freqtrade.data.history.get_timerange', get_timerange) + patch_exchange(mocker) + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') + mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=[])) + + default_conf['ticker_interval'] = "1m" + default_conf['datadir'] = testdatadir + default_conf['export'] = None + default_conf['timerange'] = '20180101-20180102' + + with pytest.raises(OperationalException, match='No pair in whitelist.'): + Backtesting(default_conf) + + default_conf['pairlists'] = [{"method": "VolumePairList", "number_assets": 5}] + with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting.'): + Backtesting(default_conf) + + def test_backtest(default_conf, fee, mocker, testdatadir) -> None: default_conf['ask_strategy']['use_sell_signal'] = False mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) @@ -524,17 +409,15 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None: timerange = TimeRange('date', None, 1517227800, 0) data = history.load_data(datadir=testdatadir, timeframe='5m', pairs=['UNITTEST/BTC'], timerange=timerange) - data_processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timerange(data_processed) + processed = backtesting.strategy.ohlcvdata_to_dataframe(data) + min_date, max_date = get_timerange(processed) results = backtesting.backtest( - { - 'stake_amount': default_conf['stake_amount'], - 'processed': data_processed, - 'max_open_trades': 10, - 'position_stacking': False, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=default_conf['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=10, + position_stacking=False, ) assert not results.empty assert len(results) == 2 @@ -557,7 +440,7 @@ def test_backtest(default_conf, fee, mocker, testdatadir) -> None: 'sell_reason': [SellType.ROI, SellType.ROI] }) pd.testing.assert_frame_equal(results, expected) - data_pair = data_processed[pair] + data_pair = processed[pair] for _, t in results.iterrows(): ln = data_pair.loc[data_pair["date"] == t["open_time"]] # Check open trade rate alignes to open rate @@ -580,17 +463,15 @@ def test_backtest_1min_ticker_interval(default_conf, fee, mocker, testdatadir) - timerange = TimeRange.parse_timerange('1510688220-1510700340') data = history.load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC'], timerange=timerange) - processed = backtesting.strategy.tickerdata_to_dataframe(data) + processed = backtesting.strategy.ohlcvdata_to_dataframe(data) min_date, max_date = get_timerange(processed) results = backtesting.backtest( - { - 'stake_amount': default_conf['stake_amount'], - 'processed': processed, - 'max_open_trades': 1, - 'position_stacking': False, - 'start_date': min_date, - 'end_date': max_date, - } + processed=processed, + stake_amount=default_conf['stake_amount'], + start_date=min_date, + end_date=max_date, + max_open_trades=1, + position_stacking=False, ) assert not results.empty assert len(results) == 1 @@ -601,7 +482,7 @@ def test_processed(default_conf, mocker, testdatadir) -> None: backtesting = Backtesting(default_conf) dict_of_tickerrows = load_data_test('raise', testdatadir) - dataframes = backtesting.strategy.tickerdata_to_dataframe(dict_of_tickerrows) + dataframes = backtesting.strategy.ohlcvdata_to_dataframe(dict_of_tickerrows) dataframe = dataframes['UNITTEST/BTC'] cols = dataframe.columns # assert the dataframe got some of the indicator columns @@ -630,7 +511,7 @@ def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) assert results.empty @@ -645,21 +526,19 @@ def test_backtest_only_sell(mocker, default_conf, testdatadir): backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = fun # Override backtesting.strategy.advise_sell = fun # Override - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) assert results.empty def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - mocker.patch('freqtrade.optimize.backtesting.file_dump_json', MagicMock()) backtest_conf = _make_backtest_conf(mocker, conf=default_conf, pair='UNITTEST/BTC', datadir=testdatadir) default_conf['ticker_interval'] = '1m' backtesting = Backtesting(default_conf) backtesting.strategy.advise_buy = _trend_alternate # Override backtesting.strategy.advise_sell = _trend_alternate # Override - results = backtesting.backtest(backtest_conf) - backtesting._store_backtest_result("test_.json", results) + results = backtesting.backtest(**backtest_conf) # 200 candles in backtest data # won't buy on first (shifted by 1) # 100 buys signals @@ -676,7 +555,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) """ Buy every xth candle - sell every other xth -2 (hold on to pairs a bit) """ - if metadata['pair'] in('ETH/BTC', 'LTC/BTC'): + if metadata['pair'] in ('ETH/BTC', 'LTC/BTC'): multi = 20 else: multi = 18 @@ -700,18 +579,18 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) backtesting.strategy.advise_buy = _trend_alternate_hold # Override backtesting.strategy.advise_sell = _trend_alternate_hold # Override - data_processed = backtesting.strategy.tickerdata_to_dataframe(data) - min_date, max_date = get_timerange(data_processed) + processed = backtesting.strategy.ohlcvdata_to_dataframe(data) + min_date, max_date = get_timerange(processed) backtest_conf = { + 'processed': processed, 'stake_amount': default_conf['stake_amount'], - 'processed': data_processed, - 'max_open_trades': 3, - 'position_stacking': False, 'start_date': min_date, 'end_date': max_date, + 'max_open_trades': 3, + 'position_stacking': False, } - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) # Make sure we have parallel trades assert len(evaluate_result_multi(results, '5m', 2)) > 0 @@ -719,101 +598,24 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) assert len(evaluate_result_multi(results, '5m', 3)) == 0 backtest_conf = { + 'processed': processed, 'stake_amount': default_conf['stake_amount'], - 'processed': data_processed, - 'max_open_trades': 1, - 'position_stacking': False, 'start_date': min_date, 'end_date': max_date, + 'max_open_trades': 1, + 'position_stacking': False, } - results = backtesting.backtest(backtest_conf) + results = backtesting.backtest(**backtest_conf) assert len(evaluate_result_multi(results, '5m', 1)) == 0 -def test_backtest_record(default_conf, fee, mocker): - names = [] - records = [] - patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - mocker.patch( - 'freqtrade.optimize.backtesting.file_dump_json', - new=lambda n, r: (names.append(n), records.append(r)) - ) - - backtesting = Backtesting(default_conf) - results = pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", - "UNITTEST/BTC", "UNITTEST/BTC"], - "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780], - "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], - "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime, - Arrow(2017, 11, 14, 21, 36, 00).datetime, - Arrow(2017, 11, 14, 22, 12, 00).datetime, - Arrow(2017, 11, 14, 22, 44, 00).datetime], - "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime, - Arrow(2017, 11, 14, 22, 10, 00).datetime, - Arrow(2017, 11, 14, 22, 43, 00).datetime, - Arrow(2017, 11, 14, 22, 58, 00).datetime], - "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], - "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], - "open_index": [1, 119, 153, 185], - "close_index": [118, 151, 184, 199], - "trade_duration": [123, 34, 31, 14], - "open_at_end": [False, False, False, True], - "sell_reason": [SellType.ROI, SellType.STOP_LOSS, - SellType.ROI, SellType.FORCE_SELL] - }) - backtesting._store_backtest_result("backtest-result.json", results) - assert len(results) == 4 - # Assert file_dump_json was only called once - assert names == ['backtest-result.json'] - records = records[0] - # Ensure records are of correct type - assert len(records) == 4 - - # reset test to test with strategy name - names = [] - records = [] - backtesting._store_backtest_result(Path("backtest-result.json"), results, "DefStrat") - assert len(results) == 4 - # Assert file_dump_json was only called once - assert names == [Path('backtest-result-DefStrat.json')] - records = records[0] - # Ensure records are of correct type - assert len(records) == 4 - - # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) - # Below follows just a typecheck of the schema/type of trade-records - oix = None - for (pair, profit, date_buy, date_sell, buy_index, dur, - openr, closer, open_at_end, sell_reason) in records: - assert pair == 'UNITTEST/BTC' - assert isinstance(profit, float) - # FIX: buy/sell should be converted to ints - assert isinstance(date_buy, float) - assert isinstance(date_sell, float) - assert isinstance(openr, float) - assert isinstance(closer, float) - assert isinstance(open_at_end, bool) - assert isinstance(sell_reason, str) - isinstance(buy_index, pd._libs.tslib.Timestamp) - if oix: - assert buy_index > oix - oix = buy_index - assert dur > 0 - - def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): - default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - async def load_pairs(pair, timeframe, since): - return _load_pair_as_ticks(pair, timeframe) - - api_mock = MagicMock() - api_mock.fetch_ohlcv = load_pairs - - patch_exchange(mocker, api_mock) + patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', MagicMock()) - mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', MagicMock()) + mocker.patch('freqtrade.optimize.backtesting.show_backtest_results', MagicMock()) + mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) patched_configuration_load_config_file(mocker, default_conf) args = [ @@ -847,21 +649,18 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): assert log_has(line, caplog) +@pytest.mark.filterwarnings("ignore:deprecated") def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): - default_conf['exchange']['pair_whitelist'] = ['UNITTEST/BTC'] - async def load_pairs(pair, timeframe, since): - return _load_pair_as_ticks(pair, timeframe) - api_mock = MagicMock() - api_mock.fetch_ohlcv = load_pairs - - patch_exchange(mocker, api_mock) + patch_exchange(mocker) backtestmock = MagicMock() + mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + PropertyMock(return_value=['UNITTEST/BTC'])) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) gen_table_mock = MagicMock() - mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table', gen_table_mock) + mocker.patch('freqtrade.optimize.optimize_reports.generate_text_table', gen_table_mock) gen_strattable_mock = MagicMock() - mocker.patch('freqtrade.optimize.backtesting.Backtesting._generate_text_table_strategy', + mocker.patch('freqtrade.optimize.optimize_reports.generate_text_table_strategy', gen_strattable_mock) patched_configuration_load_config_file(mocker, default_conf) @@ -869,14 +668,14 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'backtesting', '--config', 'config.json', '--datadir', str(testdatadir), - '--strategy-path', str(Path(__file__).parents[2] / 'freqtrade/templates'), + '--strategy-path', str(Path(__file__).parents[1] / 'strategy/strats'), '--ticker-interval', '1m', '--timerange', '1510694220-1510700340', '--enable-position-stacking', '--disable-max-market-positions', '--strategy-list', 'DefaultStrategy', - 'SampleStrategy', + 'TestStrategyLegacy', ] args = get_args(args) start_backtesting(args) @@ -899,7 +698,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): 'up to 2017-11-14T22:58:00+00:00 (0 days)..', 'Parameter --enable-position-stacking detected ...', 'Running backtesting for Strategy DefaultStrategy', - 'Running backtesting for Strategy SampleStrategy', + 'Running backtesting for Strategy TestStrategyLegacy', ] for line in exists: diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index ddfa7156e..a5e468542 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -3,15 +3,14 @@ from unittest.mock import MagicMock -from freqtrade.edge import PairInfo -from freqtrade.optimize import setup_configuration, start_edge +from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge from freqtrade.optimize.edge_cli import EdgeCli from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) -def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> None: +def test_setup_optimize_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) args = [ @@ -20,7 +19,7 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> '--strategy', 'DefaultStrategy', ] - config = setup_configuration(get_args(args), RunMode.EDGE) + config = setup_optimize_configuration(get_args(args), RunMode.EDGE) assert config['runmode'] == RunMode.EDGE assert 'max_open_trades' in config @@ -54,7 +53,7 @@ def test_setup_edge_configuration_with_arguments(mocker, edge_conf, caplog) -> N '--stoplosses=-0.01,-0.10,-0.001' ] - config = setup_configuration(get_args(args), RunMode.EDGE) + config = setup_optimize_configuration(get_args(args), RunMode.EDGE) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -83,8 +82,8 @@ def test_start(mocker, fee, edge_conf, caplog) -> None: '--config', 'config.json', '--strategy', 'DefaultStrategy', ] - args = get_args(args) - start_edge(args) + pargs = get_args(args) + start_edge(pargs) assert log_has('Starting freqtrade in Edge mode', caplog) assert start_mock.call_count == 1 @@ -106,16 +105,3 @@ def test_edge_init_fee(mocker, edge_conf) -> None: edge_cli = EdgeCli(edge_conf) assert edge_cli.edge.fee == 0.1234 assert fee_mock.call_count == 0 - - -def test_generate_edge_table(edge_conf, mocker): - patch_exchange(mocker) - edge_cli = EdgeCli(edge_conf) - - results = {} - results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) - - assert edge_cli._generate_edge_table(results).count(':|') == 7 - assert edge_cli._generate_edge_table(results).count('| ETH/BTC |') == 1 - assert edge_cli._generate_edge_table(results).count( - '| risk reward ratio | required risk reward | expectancy |') == 1 diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 29b8b5b16..90e047954 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,7 +1,9 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 import locale +import logging from datetime import datetime from pathlib import Path +from typing import Dict, List from unittest.mock import MagicMock, PropertyMock import pandas as pd @@ -9,10 +11,11 @@ import pytest from arrow import Arrow from filelock import Timeout -from freqtrade import OperationalException -from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.history import load_tickerdata_file -from freqtrade.optimize import setup_configuration, start_hyperopt +from freqtrade import constants +from freqtrade.commands.optimize_commands import (setup_optimize_configuration, + start_hyperopt) +from freqtrade.data.history import load_data +from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.optimize.default_hyperopt import DefaultHyperOpt from freqtrade.optimize.default_hyperopt_loss import DefaultHyperOptLoss from freqtrade.optimize.hyperopt import Hyperopt @@ -42,20 +45,26 @@ def hyperopt_results(): 'profit_percent': [-0.1, 0.2, 0.3], 'profit_abs': [-0.2, 0.4, 0.6], 'trade_duration': [10, 30, 10], - 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI] + 'sell_reason': [SellType.STOP_LOSS, SellType.ROI, SellType.ROI], + 'close_time': + [ + datetime(2019, 1, 1, 9, 26, 3, 478039), + datetime(2019, 2, 1, 9, 26, 3, 478039), + datetime(2019, 3, 1, 9, 26, 3, 478039) + ] } ) # Functions for recurrent object patching -def create_trials(mocker, hyperopt, testdatadir) -> None: +def create_results(mocker, hyperopt, testdatadir) -> List[Dict]: """ - When creating trials, mock the hyperopt Trials so that *by default* + When creating results, mock the hyperopt so that *by default* - we don't create any pickle'd files in the filesystem - we might have a pickle'd file so make sure that we return false when looking for it """ - hyperopt.trials_file = testdatadir / 'optimize/ut_trials.pickle' + hyperopt.results_file = testdatadir / 'optimize/ut_results.pickle' mocker.patch.object(Path, "is_file", MagicMock(return_value=False)) stat_mock = MagicMock() @@ -77,7 +86,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca '--hyperopt', 'DefaultHyperOpt', ] - config = setup_configuration(get_args(args), RunMode.HYPEROPT) + config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -117,7 +126,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo '--print-all' ] - config = setup_configuration(get_args(args), RunMode.HYPEROPT) + config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) assert 'max_open_trades' in config assert 'stake_currency' in config assert 'stake_amount' in config @@ -151,6 +160,21 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo assert log_has('Parameter --print-all detected ...', caplog) +def test_setup_hyperopt_configuration_unlimited_stake_amount(mocker, default_conf, caplog) -> None: + default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + + patched_configuration_load_config_file(mocker, default_conf) + + args = [ + 'hyperopt', + '--config', 'config.json', + '--hyperopt', 'DefaultHyperOpt', + ] + + with pytest.raises(DependencyException, match=r'.`stake_amount`.*'): + setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) + + def test_hyperoptresolver(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) @@ -159,11 +183,11 @@ def test_hyperoptresolver(mocker, default_conf, caplog) -> None: delattr(hyperopt, 'populate_buy_trend') delattr(hyperopt, 'populate_sell_trend') mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver._load_hyperopt', + 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver.load_object', MagicMock(return_value=hyperopt(default_conf)) ) default_conf.update({'hyperopt': 'DefaultHyperOpt'}) - x = HyperOptResolver(default_conf).hyperopt + x = HyperOptResolver.load_hyperopt(default_conf) assert not hasattr(x, 'populate_indicators') assert not hasattr(x, 'populate_buy_trend') assert not hasattr(x, 'populate_sell_trend') @@ -180,7 +204,7 @@ def test_hyperoptresolver_wrongname(mocker, default_conf, caplog) -> None: default_conf.update({'hyperopt': "NonExistingHyperoptClass"}) with pytest.raises(OperationalException, match=r'Impossible to load Hyperopt.*'): - HyperOptResolver(default_conf).hyperopt + HyperOptResolver.load_hyperopt(default_conf) def test_hyperoptresolver_noname(default_conf): @@ -188,17 +212,17 @@ def test_hyperoptresolver_noname(default_conf): with pytest.raises(OperationalException, match="No Hyperopt set. Please use `--hyperopt` to specify " "the Hyperopt class to use."): - HyperOptResolver(default_conf) + HyperOptResolver.load_hyperopt(default_conf) def test_hyperoptlossresolver(mocker, default_conf, caplog) -> None: hl = DefaultHyperOptLoss mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver._load_hyperoptloss', + 'freqtrade.resolvers.hyperopt_resolver.HyperOptLossResolver.load_object', MagicMock(return_value=hl) ) - x = HyperOptLossResolver(default_conf).hyperoptloss + x = HyperOptLossResolver.load_hyperoptloss(default_conf) assert hasattr(x, "hyperopt_loss_function") @@ -206,7 +230,7 @@ def test_hyperoptlossresolver_wrongname(mocker, default_conf, caplog) -> None: default_conf.update({'hyperopt_loss': "NonExistingLossClass"}) with pytest.raises(OperationalException, match=r'Impossible to load HyperoptLoss.*'): - HyperOptLossResolver(default_conf).hyperopt + HyperOptLossResolver.load_hyperoptloss(default_conf) def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None: @@ -222,10 +246,10 @@ def test_start_not_installed(mocker, default_conf, caplog, import_fails) -> None '--hyperopt', 'DefaultHyperOpt', '--epochs', '5' ] - args = get_args(args) + pargs = get_args(args) with pytest.raises(OperationalException, match=r"Please ensure that the hyperopt dependencies"): - start_hyperopt(args) + start_hyperopt(pargs) def test_start(mocker, default_conf, caplog) -> None: @@ -240,8 +264,8 @@ def test_start(mocker, default_conf, caplog) -> None: '--hyperopt', 'DefaultHyperOpt', '--epochs', '5' ] - args = get_args(args) - start_hyperopt(args) + pargs = get_args(args) + start_hyperopt(pargs) assert log_has('Starting freqtrade in Hyperopt mode', caplog) assert start_mock.call_count == 1 @@ -263,9 +287,9 @@ def test_start_no_data(mocker, default_conf, caplog) -> None: '--hyperopt', 'DefaultHyperOpt', '--epochs', '5' ] - args = get_args(args) + pargs = get_args(args) with pytest.raises(OperationalException, match='No data found. Terminating.'): - start_hyperopt(args) + start_hyperopt(pargs) def test_start_filelock(mocker, default_conf, caplog) -> None: @@ -280,16 +304,19 @@ def test_start_filelock(mocker, default_conf, caplog) -> None: '--hyperopt', 'DefaultHyperOpt', '--epochs', '5' ] - args = get_args(args) - start_hyperopt(args) + pargs = get_args(args) + start_hyperopt(pargs) assert log_has("Another running instance of freqtrade Hyperopt detected.", caplog) def test_loss_calculation_prefer_correct_trade_count(default_conf, hyperopt_results) -> None: - hl = HyperOptLossResolver(default_conf).hyperoptloss - correct = hl.hyperopt_loss_function(hyperopt_results, 600) - over = hl.hyperopt_loss_function(hyperopt_results, 600 + 100) - under = hl.hyperopt_loss_function(hyperopt_results, 600 - 100) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(hyperopt_results, 600 + 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(hyperopt_results, 600 - 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) assert over > correct assert under > correct @@ -298,9 +325,11 @@ def test_loss_calculation_prefer_shorter_trades(default_conf, hyperopt_results) resultsb = hyperopt_results.copy() resultsb.loc[1, 'trade_duration'] = 20 - hl = HyperOptLossResolver(default_conf).hyperoptloss - longer = hl.hyperopt_loss_function(hyperopt_results, 100) - shorter = hl.hyperopt_loss_function(resultsb, 100) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + longer = hl.hyperopt_loss_function(hyperopt_results, 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + shorter = hl.hyperopt_loss_function(resultsb, 100, + datetime(2019, 1, 1), datetime(2019, 5, 1)) assert shorter < longer @@ -310,10 +339,13 @@ def test_loss_calculation_has_limited_profit(default_conf, hyperopt_results) -> results_under = hyperopt_results.copy() results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 - hl = HyperOptLossResolver(default_conf).hyperoptloss - correct = hl.hyperopt_loss_function(hyperopt_results, 600) - over = hl.hyperopt_loss_function(results_over, 600) - under = hl.hyperopt_loss_function(results_under, 600) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, 600, + datetime(2019, 1, 1), datetime(2019, 5, 1)) assert over < correct assert under > correct @@ -325,7 +357,61 @@ def test_sharpe_loss_prefers_higher_profits(default_conf, hyperopt_results) -> N results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 default_conf.update({'hyperopt_loss': 'SharpeHyperOptLoss'}) - hl = HyperOptLossResolver(default_conf).hyperoptloss + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sharpe_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SharpeHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sortino_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLoss'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) + correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + under = hl.hyperopt_loss_function(results_under, len(hyperopt_results), + datetime(2019, 1, 1), datetime(2019, 5, 1)) + assert over < correct + assert under > correct + + +def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_results) -> None: + results_over = hyperopt_results.copy() + results_over['profit_percent'] = hyperopt_results['profit_percent'] * 2 + results_under = hyperopt_results.copy() + results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 + + default_conf.update({'hyperopt_loss': 'SortinoHyperOptLossDaily'}) + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), datetime(2019, 1, 1), datetime(2019, 5, 1)) over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), @@ -343,7 +429,7 @@ def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) results_under['profit_percent'] = hyperopt_results['profit_percent'] / 2 default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) - hl = HyperOptLossResolver(default_conf).hyperoptloss + hl = HyperOptLossResolver.load_hyperoptloss(default_conf) correct = hl.hyperopt_loss_function(hyperopt_results, len(hyperopt_results), datetime(2019, 1, 1), datetime(2019, 5, 1)) over = hl.hyperopt_loss_function(results_over, len(hyperopt_results), @@ -357,17 +443,27 @@ def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) def test_log_results_if_loss_improves(hyperopt, capsys) -> None: hyperopt.current_best_loss = 2 hyperopt.total_epochs = 2 + hyperopt.print_results( { - 'is_best': True, 'loss': 1, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + }, + 'total_profit': 0, 'current_epoch': 2, # This starts from 1 (in a human-friendly manner) - 'results_explanation': 'foo.', - 'is_initial_point': False + 'is_initial_point': False, + 'is_best': True } ) out, err = capsys.readouterr() - assert ' 2/2: foo. Objective: 1.00000' in out + assert all(x in out + for x in ["Best", "2/2", " 1", "0.10%", "0.00100000 BTC (1.00%)", "20.0 m"]) def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: @@ -382,30 +478,30 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_save_trials_saves_trials(mocker, hyperopt, testdatadir, caplog) -> None: - trials = create_trials(mocker, hyperopt, testdatadir) +def test_save_results_saves_epochs(mocker, hyperopt, testdatadir, caplog) -> None: + epochs = create_results(mocker, hyperopt, testdatadir) mock_dump = mocker.patch('freqtrade.optimize.hyperopt.dump', return_value=None) - trials_file = testdatadir / 'optimize' / 'ut_trials.pickle' + results_file = testdatadir / 'optimize' / 'ut_results.pickle' - hyperopt.trials = trials - hyperopt.save_trials(final=True) - assert log_has("Saving 1 epoch.", caplog) - assert log_has(f"1 epoch saved to '{trials_file}'.", caplog) + caplog.set_level(logging.DEBUG) + + hyperopt.epochs = epochs + hyperopt._save_results() + assert log_has(f"1 epoch saved to '{results_file}'.", caplog) mock_dump.assert_called_once() - hyperopt.trials = trials + trials - hyperopt.save_trials(final=True) - assert log_has("Saving 2 epochs.", caplog) - assert log_has(f"2 epochs saved to '{trials_file}'.", caplog) + hyperopt.epochs = epochs + epochs + hyperopt._save_results() + assert log_has(f"2 epochs saved to '{results_file}'.", caplog) -def test_read_trials_returns_trials_file(mocker, hyperopt, testdatadir, caplog) -> None: - trials = create_trials(mocker, hyperopt, testdatadir) - mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=trials) - trials_file = testdatadir / 'optimize' / 'ut_trials.pickle' - hyperopt_trial = hyperopt._read_trials(trials_file) - assert log_has(f"Reading Trials from '{trials_file}'", caplog) - assert hyperopt_trial == trials +def test_read_results_returns_epochs(mocker, hyperopt, testdatadir, caplog) -> None: + epochs = create_results(mocker, hyperopt, testdatadir) + mock_load = mocker.patch('freqtrade.optimize.hyperopt.load', return_value=epochs) + results_file = testdatadir / 'optimize' / 'ut_results.pickle' + hyperopt_epochs = hyperopt._read_results(results_file) + assert log_has(f"Reading epochs from '{results_file}'", caplog) + assert hyperopt_epochs == epochs mock_load.assert_called_once() @@ -433,11 +529,21 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', - 'params': {'buy': {}, 'sell': {}, 'roi': {}, 'stoploss': 0.0}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', + 'params': {'buy': {}, 'sell': {}, 'roi': {}, 'stoploss': 0.0}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + }, + }]) ) patch_exchange(mocker) - # Co-test loading ticker-interval from strategy + # Co-test loading timeframe from strategy del default_conf['ticker_interval'] default_conf.update({'config': 'config.json.example', 'hyperopt': 'DefaultHyperOpt', @@ -447,7 +553,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -457,7 +563,7 @@ def test_start_calls_optimizer(mocker, default_conf, caplog, capsys) -> None: out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for tickerdata, once to save evaluations + # Should be called twice, once for historical candle data, once to save evaluations assert dumper.call_count == 2 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") @@ -543,10 +649,8 @@ def test_has_space(hyperopt, spaces, expected_results): def test_populate_indicators(hyperopt, testdatadir) -> None: - tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", - fill_missing=True)} - dataframes = hyperopt.backtesting.strategy.tickerdata_to_dataframe(tickerlist) + data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) + dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -557,10 +661,8 @@ def test_populate_indicators(hyperopt, testdatadir) -> None: def test_buy_strategy_generator(hyperopt, testdatadir) -> None: - tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m') - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", - fill_missing=True)} - dataframes = hyperopt.backtesting.strategy.tickerdata_to_dataframe(tickerlist) + data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) + dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data) dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], {'pair': 'UNITTEST/BTC'}) @@ -700,7 +802,7 @@ def test_clean_hyperopt(mocker, default_conf, caplog): h = Hyperopt(default_conf) assert unlinkmock.call_count == 2 - assert log_has(f"Removing `{h.tickerdata_pickle}`.", caplog) + assert log_has(f"Removing `{h.data_pickle_file}`.", caplog) def test_continue_hyperopt(mocker, default_conf, caplog): @@ -718,7 +820,7 @@ def test_continue_hyperopt(mocker, default_conf, caplog): Hyperopt(default_conf) assert unlinkmock.call_count == 0 - assert log_has(f"Continuing on previous hyperopt results.", caplog) + assert log_has("Continuing on previous hyperopt results.", caplog) def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: @@ -732,11 +834,23 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}, - 'params_details': {'buy': {'mfi-value': None}, - 'sell': {'sell-mfi-value': None}, - 'roi': {}, 'stoploss': {'stoploss': None}, - 'trailing': {'trailing_stop': None}}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'params_details': { + 'buy': {'mfi-value': None}, + 'sell': {'sell-mfi-value': None}, + 'roi': {}, 'stoploss': {'stoploss': None}, + 'trailing': {'trailing_stop': None} + }, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -750,7 +864,7 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: }) hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -758,9 +872,13 @@ def test_print_json_spaces_all(mocker, default_conf, caplog, capsys) -> None: parallel.assert_called_once() out, err = capsys.readouterr() - assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null,"trailing_stop":null}' in out # noqa: E501 + result_str = ( + '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi"' + ':{},"stoploss":null,"trailing_stop":null}' + ) + assert result_str in out # noqa: E501 assert dumper.called - # Should be called twice, once for tickerdata, once to save evaluations + # Should be called twice, once for historical candle data, once to save evaluations assert dumper.call_count == 2 @@ -775,10 +893,22 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}, - 'params_details': {'buy': {'mfi-value': None}, - 'sell': {'sell-mfi-value': None}, - 'roi': {}, 'stoploss': {'stoploss': None}}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'params_details': { + 'buy': {'mfi-value': None}, + 'sell': {'sell-mfi-value': None}, + 'roi': {}, 'stoploss': {'stoploss': None} + }, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -792,7 +922,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None }) hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -802,7 +932,7 @@ def test_print_json_spaces_default(mocker, default_conf, caplog, capsys) -> None out, err = capsys.readouterr() assert '{"params":{"mfi-value":null,"sell-mfi-value":null},"minimal_roi":{},"stoploss":null}' in out # noqa: E501 assert dumper.called - # Should be called twice, once for tickerdata, once to save evaluations + # Should be called twice, once for historical candle data, once to save evaluations assert dumper.call_count == 2 @@ -817,8 +947,18 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}, - 'params_details': {'roi': {}, 'stoploss': {'stoploss': None}}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'params_details': {'roi': {}, 'stoploss': {'stoploss': None}}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -832,7 +972,7 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> }) hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) hyperopt.start() @@ -842,7 +982,7 @@ def test_print_json_spaces_roi_stoploss(mocker, default_conf, caplog, capsys) -> out, err = capsys.readouterr() assert '{"minimal_roi":{},"stoploss":null}' in out assert dumper.called - # Should be called twice, once for tickerdata, once to save evaluations + # Should be called twice, once for historical candle data, once to save evaluations assert dumper.call_count == 2 @@ -858,7 +998,16 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', MagicMock(return_value=[{ - 'loss': 1, 'results_explanation': 'foo result', 'params': {'stoploss': 0.0}}]) + 'loss': 1, 'results_explanation': 'foo result', 'params': {'stoploss': 0.0}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -870,7 +1019,7 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) del hyperopt.custom_hyperopt.__class__.buy_strategy_generator @@ -885,7 +1034,7 @@ def test_simplified_interface_roi_stoploss(mocker, default_conf, caplog, capsys) out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for tickerdata, once to save evaluations + # Should be called twice, once for historical candle data, once to save evaluations assert dumper.call_count == 2 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") @@ -913,7 +1062,7 @@ def test_simplified_interface_all_failed(mocker, default_conf, caplog, capsys) - 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) del hyperopt.custom_hyperopt.__class__.buy_strategy_generator @@ -936,7 +1085,17 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -948,7 +1107,7 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) # TODO: sell_strategy_generator() is actually not called because @@ -963,7 +1122,7 @@ def test_simplified_interface_buy(mocker, default_conf, caplog, capsys) -> None: out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for tickerdata, once to save evaluations + # Should be called twice, once for historical candle data, once to save evaluations assert dumper.call_count == 2 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") @@ -983,7 +1142,17 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None parallel = mocker.patch( 'freqtrade.optimize.hyperopt.Hyperopt.run_optimizer_parallel', - MagicMock(return_value=[{'loss': 1, 'results_explanation': 'foo result', 'params': {}}]) + MagicMock(return_value=[{ + 'loss': 1, 'results_explanation': 'foo result', 'params': {}, + 'results_metrics': + { + 'trade_count': 1, + 'avg_profit': 0.1, + 'total_profit': 0.001, + 'profit': 1.0, + 'duration': 20.0 + } + }]) ) patch_exchange(mocker) @@ -995,7 +1164,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) # TODO: buy_strategy_generator() is actually not called because @@ -1010,7 +1179,7 @@ def test_simplified_interface_sell(mocker, default_conf, caplog, capsys) -> None out, err = capsys.readouterr() assert 'Best result:\n\n* 1/1: foo result Objective: 1.00000\n' in out assert dumper.called - # Should be called twice, once for tickerdata, once to save evaluations + # Should be called twice, once for historical candle data, once to save evaluations assert dumper.call_count == 2 assert hasattr(hyperopt.backtesting.strategy, "advise_sell") assert hasattr(hyperopt.backtesting.strategy, "advise_buy") @@ -1044,7 +1213,7 @@ def test_simplified_interface_failed(mocker, default_conf, caplog, capsys, metho 'hyperopt_jobs': 1, }) hyperopt = Hyperopt(default_conf) - hyperopt.backtesting.strategy.tickerdata_to_dataframe = MagicMock() + hyperopt.backtesting.strategy.ohlcvdata_to_dataframe = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) delattr(hyperopt.custom_hyperopt.__class__, method) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py new file mode 100644 index 000000000..e0782146a --- /dev/null +++ b/tests/optimize/test_optimize_reports.py @@ -0,0 +1,193 @@ +from pathlib import Path + +import pandas as pd +from arrow import Arrow + +from freqtrade.edge import PairInfo +from freqtrade.optimize.optimize_reports import ( + generate_edge_table, generate_text_table, generate_text_table_sell_reason, + generate_text_table_strategy, store_backtest_result) +from freqtrade.strategy.interface import SellType +from tests.conftest import patch_exchange + + +def test_generate_text_table(default_conf, mocker): + + results = pd.DataFrame( + { + 'pair': ['ETH/BTC', 'ETH/BTC'], + 'profit_percent': [0.1, 0.2], + 'profit_abs': [0.2, 0.4], + 'trade_duration': [10, 30], + 'wins': [2, 0], + 'draws': [0, 0], + 'losses': [0, 0] + } + ) + + result_str = ( + '| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC |' + ' Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|---------+--------+----------------+----------------+------------------+' + '----------------+----------------+--------+---------+----------|\n' + '| ETH/BTC | 2 | 15.00 | 30.00 | 0.60000000 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |\n' + '| TOTAL | 2 | 15.00 | 30.00 | 0.60000000 |' + ' 15.00 | 0:20:00 | 2 | 0 | 0 |' + ) + assert generate_text_table(data={'ETH/BTC': {}}, + stake_currency='BTC', max_open_trades=2, + results=results) == result_str + + +def test_generate_text_table_sell_reason(default_conf, mocker): + + results = pd.DataFrame( + { + 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'profit_percent': [0.1, 0.2, -0.1], + 'profit_abs': [0.2, 0.4, -0.2], + 'trade_duration': [10, 30, 10], + 'wins': [2, 0, 0], + 'draws': [0, 0, 0], + 'losses': [0, 0, 1], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + } + ) + + result_str = ( + '| Sell Reason | Sells | Wins | Draws | Losses |' + ' Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % |\n' + '|---------------+---------+--------+---------+----------+' + '----------------+----------------+------------------+----------------|\n' + '| roi | 2 | 2 | 0 | 0 |' + ' 15 | 30 | 0.6 | 15 |\n' + '| stop_loss | 1 | 0 | 0 | 1 |' + ' -10 | -10 | -0.2 | -5 |' + ) + assert generate_text_table_sell_reason(stake_currency='BTC', max_open_trades=2, + results=results) == result_str + + +def test_generate_text_table_strategy(default_conf, mocker): + results = {} + results['TestStrategy1'] = pd.DataFrame( + { + 'pair': ['ETH/BTC', 'ETH/BTC', 'ETH/BTC'], + 'profit_percent': [0.1, 0.2, 0.3], + 'profit_abs': [0.2, 0.4, 0.5], + 'trade_duration': [10, 30, 10], + 'wins': [2, 0, 0], + 'draws': [0, 0, 0], + 'losses': [0, 0, 1], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + } + ) + results['TestStrategy2'] = pd.DataFrame( + { + 'pair': ['LTC/BTC', 'LTC/BTC', 'LTC/BTC'], + 'profit_percent': [0.4, 0.2, 0.3], + 'profit_abs': [0.4, 0.4, 0.5], + 'trade_duration': [15, 30, 15], + 'wins': [4, 1, 0], + 'draws': [0, 0, 0], + 'losses': [0, 0, 1], + 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] + } + ) + + result_str = ( + '| Strategy | Buys | Avg Profit % | Cum Profit % | Tot' + ' Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |\n' + '|---------------+--------+----------------+----------------+------------------+' + '----------------+----------------+--------+---------+----------|\n' + '| TestStrategy1 | 3 | 20.00 | 60.00 | 1.10000000 |' + ' 30.00 | 0:17:00 | 3 | 0 | 0 |\n' + '| TestStrategy2 | 3 | 30.00 | 90.00 | 1.30000000 |' + ' 45.00 | 0:20:00 | 3 | 0 | 0 |' + ) + assert generate_text_table_strategy('BTC', 2, all_results=results) == result_str + + +def test_generate_edge_table(edge_conf, mocker): + + results = {} + results['ETH/BTC'] = PairInfo(-0.01, 0.60, 2, 1, 3, 10, 60) + assert generate_edge_table(results).count('+') == 7 + assert generate_edge_table(results).count('| ETH/BTC |') == 1 + assert generate_edge_table(results).count( + '| Risk Reward Ratio | Required Risk Reward | Expectancy |') == 1 + + +def test_backtest_record(default_conf, fee, mocker): + names = [] + records = [] + patch_exchange(mocker) + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + mocker.patch( + 'freqtrade.optimize.optimize_reports.file_dump_json', + new=lambda n, r: (names.append(n), records.append(r)) + ) + + results = {'DefStrat': pd.DataFrame({"pair": ["UNITTEST/BTC", "UNITTEST/BTC", + "UNITTEST/BTC", "UNITTEST/BTC"], + "profit_percent": [0.003312, 0.010801, 0.013803, 0.002780], + "profit_abs": [0.000003, 0.000011, 0.000014, 0.000003], + "open_time": [Arrow(2017, 11, 14, 19, 32, 00).datetime, + Arrow(2017, 11, 14, 21, 36, 00).datetime, + Arrow(2017, 11, 14, 22, 12, 00).datetime, + Arrow(2017, 11, 14, 22, 44, 00).datetime], + "close_time": [Arrow(2017, 11, 14, 21, 35, 00).datetime, + Arrow(2017, 11, 14, 22, 10, 00).datetime, + Arrow(2017, 11, 14, 22, 43, 00).datetime, + Arrow(2017, 11, 14, 22, 58, 00).datetime], + "open_rate": [0.002543, 0.003003, 0.003089, 0.003214], + "close_rate": [0.002546, 0.003014, 0.003103, 0.003217], + "open_index": [1, 119, 153, 185], + "close_index": [118, 151, 184, 199], + "trade_duration": [123, 34, 31, 14], + "open_at_end": [False, False, False, True], + "sell_reason": [SellType.ROI, SellType.STOP_LOSS, + SellType.ROI, SellType.FORCE_SELL] + })} + store_backtest_result(Path("backtest-result.json"), results) + # Assert file_dump_json was only called once + assert names == [Path('backtest-result.json')] + records = records[0] + # Ensure records are of correct type + assert len(records) == 4 + + # reset test to test with strategy name + names = [] + records = [] + results['Strat'] = results['DefStrat'] + results['Strat2'] = results['DefStrat'] + store_backtest_result(Path("backtest-result.json"), results) + assert names == [ + Path('backtest-result-DefStrat.json'), + Path('backtest-result-Strat.json'), + Path('backtest-result-Strat2.json'), + ] + records = records[0] + # Ensure records are of correct type + assert len(records) == 4 + + # ('UNITTEST/BTC', 0.00331158, '1510684320', '1510691700', 0, 117) + # Below follows just a typecheck of the schema/type of trade-records + oix = None + for (pair, profit, date_buy, date_sell, buy_index, dur, + openr, closer, open_at_end, sell_reason) in records: + assert pair == 'UNITTEST/BTC' + assert isinstance(profit, float) + # FIX: buy/sell should be converted to ints + assert isinstance(date_buy, float) + assert isinstance(date_sell, float) + assert isinstance(openr, float) + assert isinstance(closer, float) + assert isinstance(open_at_end, bool) + assert isinstance(sell_reason, str) + isinstance(buy_index, pd._libs.tslib.Timestamp) + if oix: + assert buy_index > oix + oix = buy_index + assert dur > 0 diff --git a/tests/pairlist/test_pairlist.py b/tests/pairlist/test_pairlist.py index 43285cdb1..61f6b4bd5 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/pairlist/test_pairlist.py @@ -4,11 +4,11 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade import OperationalException from freqtrade.constants import AVAILABLE_PAIRLISTS -from freqtrade.resolvers import PairListResolver +from freqtrade.exceptions import OperationalException from freqtrade.pairlist.pairlistmanager import PairListManager -from tests.conftest import get_patched_freqtradebot, log_has_re +from freqtrade.resolvers import PairListResolver +from tests.conftest import get_patched_freqtradebot, log_has, log_has_re # whitelist, blacklist @@ -46,44 +46,67 @@ def static_pl_conf(whitelist_conf): return whitelist_conf +def test_log_on_refresh(mocker, static_pl_conf, markets, tickers): + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) + logmock = MagicMock() + # Assign starting whitelist + pl = freqtrade.pairlists._pairlists[0] + pl.log_on_refresh(logmock, 'Hello world') + assert logmock.call_count == 1 + pl.log_on_refresh(logmock, 'Hello world') + assert logmock.call_count == 1 + assert pl._log_cache.currsize == 1 + assert ('Hello world',) in pl._log_cache._Cache__data + + pl.log_on_refresh(logmock, 'Hello world2') + assert logmock.call_count == 2 + assert pl._log_cache.currsize == 2 + + def test_load_pairlist_noexist(mocker, markets, default_conf): - bot = get_patched_freqtradebot(mocker, default_conf) + freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - plm = PairListManager(bot.exchange, default_conf) + plm = PairListManager(freqtrade.exchange, default_conf) with pytest.raises(OperationalException, match=r"Impossible to load Pairlist 'NonexistingPairList'. " r"This class does not exist or contains Python code errors."): - PairListResolver('NonexistingPairList', bot.exchange, plm, default_conf, {}, 1) + PairListResolver.load_pairlist('NonexistingPairList', freqtrade.exchange, plm, + default_conf, {}, 1) def test_refresh_market_pair_not_in_whitelist(mocker, markets, static_pl_conf): - freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf) + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) - freqtradebot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert set(whitelist) == set(freqtradebot.pairlists.whitelist) + assert set(whitelist) == set(freqtrade.pairlists.whitelist) # Ensure config dict hasn't been changed assert (static_pl_conf['exchange']['pair_whitelist'] == - freqtradebot.config['exchange']['pair_whitelist']) + freqtrade.config['exchange']['pair_whitelist']) def test_refresh_static_pairlist(mocker, markets, static_pl_conf): - freqtradebot = get_patched_freqtradebot(mocker, static_pl_conf) + freqtrade = get_patched_freqtradebot(mocker, static_pl_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', exchange_has=MagicMock(return_value=True), markets=PropertyMock(return_value=markets), ) - freqtradebot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() # List ordered by BaseVolume whitelist = ['ETH/BTC', 'TKN/BTC'] # Ensure all except those in whitelist are removed - assert set(whitelist) == set(freqtradebot.pairlists.whitelist) - assert static_pl_conf['exchange']['pair_blacklist'] == freqtradebot.pairlists.blacklist + assert set(whitelist) == set(freqtrade.pairlists.whitelist) + assert static_pl_conf['exchange']['pair_blacklist'] == freqtrade.pairlists.blacklist def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_conf): @@ -93,7 +116,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co get_tickers=tickers, exchange_has=MagicMock(return_value=True), ) - bot = get_patched_freqtradebot(mocker, whitelist_conf) + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) # Remock markets with shitcoinmarkets since get_patched_freqtradebot uses the markets fixture mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -101,9 +124,9 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co ) # argument: use the whitelist dynamically by exchange-volume whitelist = ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC'] - bot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() - assert whitelist == bot.pairlists.whitelist + assert whitelist == freqtrade.pairlists.whitelist whitelist_conf['pairlists'] = [{'method': 'VolumePairList', 'config': {} @@ -113,7 +136,7 @@ def test_refresh_pairlist_dynamic(mocker, shitcoinmarkets, tickers, whitelist_co with pytest.raises(OperationalException, match=r'`number_assets` not specified. Please check your configuration ' r'for "pairlist.config.number_assets"'): - PairListManager(bot.exchange, whitelist_conf) + PairListManager(freqtrade.exchange, whitelist_conf) def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): @@ -121,13 +144,13 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): 'freqtrade.exchange.Exchange', exchange_has=MagicMock(return_value=True), ) - freqtradebot = get_patched_freqtradebot(mocker, whitelist_conf) + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets_empty)) # argument: use the whitelist dynamically by exchange-volume whitelist = [] whitelist_conf['exchange']['pair_whitelist'] = [] - freqtradebot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() pairslist = whitelist_conf['exchange']['pair_whitelist'] assert set(whitelist) == set(pairslist) @@ -140,7 +163,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}], "BTC", ['HOT/BTC', 'FUEL/BTC', 'XRP/BTC', 'LTC/BTC', 'TKN/BTC']), ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], - "USDT", ['ETH/USDT']), + "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT']), # No pair for ETH ... ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}], "ETH", []), @@ -154,11 +177,19 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PriceFilter", "low_price_ratio": 0.03}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + # PriceFilter and VolumePairList + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "PriceFilter", "low_price_ratio": 0.03}], + "USDT", ['ETH/USDT', 'NANO/USDT']), # Hot is removed by precision_filter, Fuel by low_price_filter. ([{"method": "VolumePairList", "number_assets": 6, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.02} ], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC']), + # HOT and XRP are removed because below 1250 quoteVolume + ([{"method": "VolumePairList", "number_assets": 5, + "sort_key": "quoteVolume", "min_value": 1250}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), # StaticPairlist Only ([{"method": "StaticPairList"}, ], "BTC", ['ETH/BTC', 'TKN/BTC']), @@ -166,11 +197,16 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "bidVolume"}, ], "BTC", ['TKN/BTC', 'ETH/BTC']), + # SpreadFilter + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "SpreadFilter", "max_spread": 0.005} + ], "USDT", ['ETH/USDT']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, pairlists, base_currency, whitelist_result, caplog) -> None: whitelist_conf['pairlists'] = pairlists + whitelist_conf['stake_currency'] = base_currency mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) @@ -180,7 +216,6 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t markets=PropertyMock(return_value=shitcoinmarkets), ) - freqtrade.config['stake_currency'] = base_currency freqtrade.pairlists.refresh_pairlist() whitelist = freqtrade.pairlists.whitelist @@ -190,7 +225,16 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' r'would be <= stop limit.*', caplog) if pairlist['method'] == 'PriceFilter': - assert log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) + assert (log_has_re(r'^Removed .* from whitelist, because 1 unit is .*%$', caplog) or + log_has_re(r"^Removed .* from whitelist, because ticker\['last'\] is empty.*", + caplog)) + if pairlist['method'] == 'VolumePairList': + logmsg = ("DEPRECATED: using any key other than quoteVolume for " + "VolumePairList is deprecated.") + if pairlist['sort_key'] != 'quoteVolume': + assert log_has(logmsg, caplog) + else: + assert not log_has(logmsg, caplog) def test_gen_pair_whitelist_not_supported(mocker, default_conf, tickers) -> None: @@ -231,8 +275,6 @@ def test_pairlist_class(mocker, whitelist_conf, markets, pairlist): (['ETH/BTC', 'TKN/BTC', 'ETH/USDT'], "is not compatible with your stake currency"), # BCH/BTC not available (['ETH/BTC', 'TKN/BTC', 'BCH/BTC'], "is not compatible with exchange"), - # BLK/BTC in blacklist - (['ETH/BTC', 'TKN/BTC', 'BLK/BTC'], "in your blacklist. Removing "), # BTT/BTC is inactive (['ETH/BTC', 'TKN/BTC', 'BTT/BTC'], "Market is not active") ]) @@ -270,18 +312,18 @@ def test_volumepairlist_caching(mocker, markets, whitelist_conf, tickers): exchange_has=MagicMock(return_value=True), get_tickers=tickers ) - bot = get_patched_freqtradebot(mocker, whitelist_conf) - assert bot.pairlists._pairlists[0]._last_refresh == 0 + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + assert freqtrade.pairlists._pairlists[0]._last_refresh == 0 assert tickers.call_count == 0 - bot.pairlists.refresh_pairlist() + freqtrade.pairlists.refresh_pairlist() assert tickers.call_count == 1 - assert bot.pairlists._pairlists[0]._last_refresh != 0 - lrf = bot.pairlists._pairlists[0]._last_refresh - bot.pairlists.refresh_pairlist() + assert freqtrade.pairlists._pairlists[0]._last_refresh != 0 + lrf = freqtrade.pairlists._pairlists[0]._last_refresh + freqtrade.pairlists.refresh_pairlist() assert tickers.call_count == 1 # Time should not be updated. - assert bot.pairlists._pairlists[0]._last_refresh == lrf + assert freqtrade.pairlists._pairlists[0]._last_refresh == lrf def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): diff --git a/tests/rpc/test_fiat_convert.py b/tests/rpc/test_fiat_convert.py index 05760ce25..ed21bc516 100644 --- a/tests/rpc/test_fiat_convert.py +++ b/tests/rpc/test_fiat_convert.py @@ -8,7 +8,7 @@ import pytest from requests.exceptions import RequestException from freqtrade.rpc.fiat_convert import CryptoFiat, CryptoToFiatConverter -from tests.conftest import log_has +from tests.conftest import log_has, log_has_re def test_pair_convertion_object(): @@ -22,8 +22,8 @@ def test_pair_convertion_object(): assert pair_convertion.CACHE_DURATION == 6 * 60 * 60 # Check a regular usage - assert pair_convertion.crypto_symbol == 'BTC' - assert pair_convertion.fiat_symbol == 'USD' + assert pair_convertion.crypto_symbol == 'btc' + assert pair_convertion.fiat_symbol == 'usd' assert pair_convertion.price == 12345.0 assert pair_convertion.is_expired() is False @@ -57,15 +57,15 @@ def test_fiat_convert_add_pair(mocker): fiat_convert._add_pair(crypto_symbol='btc', fiat_symbol='usd', price=12345.0) pair_len = len(fiat_convert._pairs) assert pair_len == 1 - assert fiat_convert._pairs[0].crypto_symbol == 'BTC' - assert fiat_convert._pairs[0].fiat_symbol == 'USD' + assert fiat_convert._pairs[0].crypto_symbol == 'btc' + assert fiat_convert._pairs[0].fiat_symbol == 'usd' assert fiat_convert._pairs[0].price == 12345.0 fiat_convert._add_pair(crypto_symbol='btc', fiat_symbol='Eur', price=13000.2) pair_len = len(fiat_convert._pairs) assert pair_len == 2 - assert fiat_convert._pairs[1].crypto_symbol == 'BTC' - assert fiat_convert._pairs[1].fiat_symbol == 'EUR' + assert fiat_convert._pairs[1].crypto_symbol == 'btc' + assert fiat_convert._pairs[1].fiat_symbol == 'eur' assert fiat_convert._pairs[1].price == 13000.2 @@ -100,15 +100,15 @@ def test_fiat_convert_get_price(mocker): fiat_convert = CryptoToFiatConverter() - with pytest.raises(ValueError, match=r'The fiat US DOLLAR is not supported.'): - fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='US Dollar') + with pytest.raises(ValueError, match=r'The fiat us dollar is not supported.'): + fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='US Dollar') # Check the value return by the method pair_len = len(fiat_convert._pairs) assert pair_len == 0 - assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 28000.0 - assert fiat_convert._pairs[0].crypto_symbol == 'BTC' - assert fiat_convert._pairs[0].fiat_symbol == 'USD' + assert fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='usd') == 28000.0 + assert fiat_convert._pairs[0].crypto_symbol == 'btc' + assert fiat_convert._pairs[0].fiat_symbol == 'usd' assert fiat_convert._pairs[0].price == 28000.0 assert fiat_convert._pairs[0]._expiration != 0 assert len(fiat_convert._pairs) == 1 @@ -116,13 +116,13 @@ def test_fiat_convert_get_price(mocker): # Verify the cached is used fiat_convert._pairs[0].price = 9867.543 expiration = fiat_convert._pairs[0]._expiration - assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 9867.543 + assert fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='usd') == 9867.543 assert fiat_convert._pairs[0]._expiration == expiration # Verify the cache expiration expiration = time.time() - 2 * 60 * 60 fiat_convert._pairs[0]._expiration = expiration - assert fiat_convert.get_price(crypto_symbol='BTC', fiat_symbol='USD') == 28000.0 + assert fiat_convert.get_price(crypto_symbol='btc', fiat_symbol='usd') == 28000.0 assert fiat_convert._pairs[0]._expiration is not expiration @@ -143,15 +143,15 @@ def test_loadcryptomap(mocker): fiat_convert = CryptoToFiatConverter() assert len(fiat_convert._cryptomap) == 2 - assert fiat_convert._cryptomap["BTC"] == "1" + assert fiat_convert._cryptomap["btc"] == "bitcoin" def test_fiat_init_network_exception(mocker): # Because CryptoToFiatConverter is a Singleton we reset the listings listmock = MagicMock(side_effect=RequestException) mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.Market', - listings=listmock, + 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', + get_coins_list=listmock, ) # with pytest.raises(RequestEsxception): fiat_convert = CryptoToFiatConverter() @@ -163,24 +163,24 @@ def test_fiat_init_network_exception(mocker): def test_fiat_convert_without_network(mocker): - # Because CryptoToFiatConverter is a Singleton we reset the value of _coinmarketcap + # Because CryptoToFiatConverter is a Singleton we reset the value of _coingekko fiat_convert = CryptoToFiatConverter() - cmc_temp = CryptoToFiatConverter._coinmarketcap - CryptoToFiatConverter._coinmarketcap = None + cmc_temp = CryptoToFiatConverter._coingekko + CryptoToFiatConverter._coingekko = None - assert fiat_convert._coinmarketcap is None - assert fiat_convert._find_price(crypto_symbol='BTC', fiat_symbol='USD') == 0.0 - CryptoToFiatConverter._coinmarketcap = cmc_temp + assert fiat_convert._coingekko is None + assert fiat_convert._find_price(crypto_symbol='btc', fiat_symbol='usd') == 0.0 + CryptoToFiatConverter._coingekko = cmc_temp def test_fiat_invalid_response(mocker, caplog): # Because CryptoToFiatConverter is a Singleton we reset the listings listmock = MagicMock(return_value="{'novalidjson':DEADBEEFf}") mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.Market', - listings=listmock, + 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', + get_coins_list=listmock, ) # with pytest.raises(RequestEsxception): fiat_convert = CryptoToFiatConverter() @@ -189,8 +189,8 @@ def test_fiat_invalid_response(mocker, caplog): length_cryptomap = len(fiat_convert._cryptomap) assert length_cryptomap == 0 - assert log_has('Could not load FIAT Cryptocurrency map for the following problem: TypeError', - caplog) + assert log_has_re('Could not load FIAT Cryptocurrency map for the following problem: .*', + caplog) def test_convert_amount(mocker): diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 0a8c1cabd..63691dfb4 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -7,13 +7,13 @@ from unittest.mock import ANY, MagicMock, PropertyMock import pytest from numpy import isnan -from freqtrade import DependencyException, TemporaryError from freqtrade.edge import PairInfo +from freqtrade.exceptions import DependencyException, TemporaryError from freqtrade.persistence import Trade from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State -from tests.conftest import patch_get_signal, get_patched_freqtradebot +from tests.conftest import get_patched_freqtradebot, patch_get_signal, create_mock_trades # Functions for recurrent object patching @@ -29,7 +29,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -41,7 +41,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: with pytest.raises(RPCException, match=r'.*no active trade*'): rpc._rpc_trade_status() - freqtradebot.create_trades() + freqtradebot.enter_positions() results = rpc._rpc_trade_status() assert { 'trade_id': 1, @@ -49,15 +49,32 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'base_currency': 'BTC', 'open_date': ANY, 'open_date_hum': ANY, + 'is_open': ANY, + 'fee_open': ANY, + 'fee_open_cost': ANY, + 'fee_open_currency': ANY, + 'fee_close': ANY, + 'fee_close_cost': ANY, + 'fee_close_currency': ANY, + 'open_rate_requested': ANY, + 'open_trade_price': ANY, + 'close_rate_requested': ANY, + 'sell_reason': ANY, + 'sell_order_status': ANY, + 'min_rate': ANY, + 'max_rate': ANY, + 'strategy': ANY, + 'ticker_interval': ANY, + 'open_order_id': ANY, 'close_date': None, 'close_date_hum': None, - 'open_rate': 1.099e-05, + 'open_rate': 1.098e-05, 'close_rate': None, - 'current_rate': 1.098e-05, - 'amount': 90.99181074, + 'current_rate': 1.099e-05, + 'amount': 91.07468124, 'stake_amount': 0.001, 'close_profit': None, - 'current_profit': -0.59, + 'current_profit': -0.41, 'stop_loss': 0.0, 'initial_stop_loss': 0.0, 'initial_stop_loss_pct': None, @@ -65,10 +82,8 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'open_order': '(limit buy rem=0.00000000)' } == results[0] - mocker.patch('freqtrade.exchange.Exchange.get_ticker', - MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + MagicMock(side_effect=DependencyException("Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) assert isnan(results[0]['current_rate']) @@ -78,12 +93,29 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'base_currency': 'BTC', 'open_date': ANY, 'open_date_hum': ANY, + 'is_open': ANY, + 'fee_open': ANY, + 'fee_open_cost': ANY, + 'fee_open_currency': ANY, + 'fee_close': ANY, + 'fee_close_cost': ANY, + 'fee_close_currency': ANY, + 'open_rate_requested': ANY, + 'open_trade_price': ANY, + 'close_rate_requested': ANY, + 'sell_reason': ANY, + 'sell_order_status': ANY, + 'min_rate': ANY, + 'max_rate': ANY, + 'strategy': ANY, + 'ticker_interval': ANY, + 'open_order_id': ANY, 'close_date': None, 'close_date_hum': None, - 'open_rate': 1.099e-05, + 'open_rate': 1.098e-05, 'close_rate': None, 'current_rate': ANY, - 'amount': 90.99181074, + 'amount': 91.07468124, 'stake_amount': 0.001, 'close_profit': None, 'current_profit': ANY, @@ -97,14 +129,14 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0}), + 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', + get_price=MagicMock(return_value={'bitcoin': {'usd': 15000.0}}), ) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -113,17 +145,17 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: rpc = RPC(freqtradebot) freqtradebot.state = State.RUNNING - with pytest.raises(RPCException, match=r'.*no active order*'): + with pytest.raises(RPCException, match=r'.*no active trade*'): rpc._rpc_status_table(default_conf['stake_currency'], 'USD') - freqtradebot.create_trades() + freqtradebot.enter_positions() result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert "Since" in headers assert "Pair" in headers assert 'instantly' == result[0][2] - assert 'ETH/BTC' == result[0][1] - assert '-0.59%' == result[0][3] + assert 'ETH/BTC' in result[0][1] + assert '-0.41%' == result[0][3] # Test with fiatconvert rpc._fiat_converter = CryptoToFiatConverter() @@ -131,16 +163,14 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert "Since" in headers assert "Pair" in headers assert 'instantly' == result[0][2] - assert 'ETH/BTC' == result[0][1] - assert '-0.59% (-0.09)' == result[0][3] + assert 'ETH/BTC' in result[0][1] + assert '-0.41% (-0.06)' == result[0][3] - mocker.patch('freqtrade.exchange.Exchange.get_ticker', - MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + MagicMock(side_effect=DependencyException("Pair 'ETH/BTC' not available"))) result, headers = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] - assert 'ETH/BTC' == result[0][1] + assert 'ETH/BTC' in result[0][1] assert 'nan%' == result[0][3] @@ -149,7 +179,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -162,7 +192,7 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, rpc = RPC(freqtradebot) rpc._fiat_converter = CryptoToFiatConverter() # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.first() assert trade @@ -175,33 +205,61 @@ def test_rpc_daily_profit(default_conf, update, ticker, fee, # Try valid data update.message.text = '/daily 2' days = rpc._rpc_daily_profit(7, stake_currency, fiat_display_currency) - assert len(days) == 7 - for day in days: + assert len(days['data']) == 7 + assert days['stake_currency'] == default_conf['stake_currency'] + assert days['fiat_display_currency'] == default_conf['fiat_display_currency'] + for day in days['data']: # [datetime.date(2018, 1, 11), '0.00000000 BTC', '0.000 USD'] - assert (day[1] == '0.00000000 BTC' or - day[1] == '0.00006217 BTC') + assert (day['abs_profit'] == '0.00000000' or + day['abs_profit'] == '0.00006217') - assert (day[2] == '0.000 USD' or - day[2] == '0.933 USD') + assert (day['fiat_value'] == '0.000' or + day['fiat_value'] == '0.767') # ensure first day is current date - assert str(days[0][0]) == str(datetime.utcnow().date()) + assert str(days['data'][0]['date']) == str(datetime.utcnow().date()) # Try invalid data with pytest.raises(RPCException, match=r'.*must be an integer greater than 0*'): rpc._rpc_daily_profit(0, stake_currency, fiat_display_currency) +def test_rpc_trade_history(mocker, default_conf, markets, fee): + mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets) + ) + + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + create_mock_trades(fee) + rpc = RPC(freqtradebot) + rpc._fiat_converter = CryptoToFiatConverter() + trades = rpc._rpc_trade_history(2) + assert len(trades['trades']) == 2 + assert trades['trades_count'] == 2 + assert isinstance(trades['trades'][0], dict) + assert isinstance(trades['trades'][1], dict) + + trades = rpc._rpc_trade_history(0) + assert len(trades['trades']) == 3 + assert trades['trades_count'] == 3 + # The first trade is for ETH ... sorting is descending + assert trades['trades'][-1]['pair'] == 'ETH/BTC' + assert trades['trades'][0]['pair'] == 'ETC/BTC' + assert trades['trades'][1]['pair'] == 'ETC/BTC' + + def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, limit_buy_order, limit_sell_order, mocker) -> None: mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0}), + 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', + get_price=MagicMock(return_value={'bitcoin': {'usd': 15000.0}}), ) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -217,7 +275,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) @@ -225,13 +283,13 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() trade.is_open = False - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) @@ -239,7 +297,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() @@ -249,9 +307,9 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['profit_closed_coin'], 6.217e-05) assert prec_satoshi(stats['profit_closed_percent'], 6.2) assert prec_satoshi(stats['profit_closed_fiat'], 0.93255) - assert prec_satoshi(stats['profit_all_coin'], 5.632e-05) - assert prec_satoshi(stats['profit_all_percent'], 2.81) - assert prec_satoshi(stats['profit_all_fiat'], 0.8448) + assert prec_satoshi(stats['profit_all_coin'], 5.802e-05) + assert prec_satoshi(stats['profit_all_percent'], 2.89) + assert prec_satoshi(stats['profit_all_fiat'], 0.8703) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' @@ -260,10 +318,8 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.exchange.Exchange.get_ticker', - MagicMock(side_effect=DependencyException(f"Pair 'ETH/BTC' not available"))) - # invalidate ticker cache - rpc._freqtrade.exchange._cached_ticker = {} + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + MagicMock(side_effect=DependencyException("Pair 'ETH/BTC' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' @@ -279,15 +335,15 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, ticker_sell_up, limit_buy_order, limit_sell_order): mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0}), + 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', + get_price=MagicMock(return_value={'bitcoin': {'usd': 15000.0}}), ) mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -299,14 +355,14 @@ def test_rpc_trade_statistics_closed(mocker, default_conf, ticker, fee, rpc = RPC(freqtradebot) # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) # Update the ticker with a market going up mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up, + fetch_ticker=ticker_sell_up, get_fee=fee ) trade.update(limit_sell_order) @@ -347,8 +403,8 @@ def test_rpc_balance_handle_error(default_conf, mocker): # ETH will be skipped due to mocked Error below mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0}), + 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', + get_price=MagicMock(return_value={'bitcoin': {'usd': 15000.0}}), ) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) @@ -386,8 +442,8 @@ def test_rpc_balance_handle(default_conf, mocker, tickers): } mocker.patch.multiple( - 'freqtrade.rpc.fiat_convert.Market', - ticker=MagicMock(return_value={'price_usd': 15000.0}), + 'freqtrade.rpc.fiat_convert.CoinGeckoAPI', + get_price=MagicMock(return_value={'bitcoin': {'usd': 15000.0}}), ) mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) @@ -439,7 +495,7 @@ def test_rpc_start(mocker, default_conf) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock() + fetch_ticker=MagicMock() ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -460,7 +516,7 @@ def test_rpc_stop(mocker, default_conf) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock() + fetch_ticker=MagicMock() ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -482,7 +538,7 @@ def test_rpc_stopbuy(mocker, default_conf) -> None: mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock() + fetch_ticker=MagicMock() ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -502,7 +558,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, cancel_order=cancel_order_mock, get_order=MagicMock( return_value={ @@ -513,6 +569,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: ), get_fee=fee, ) + mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=1000) freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) @@ -529,7 +586,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: msg = rpc._rpc_forcesell('all') assert msg == {'result': 'Created sell orders for all open trades.'} - freqtradebot.create_trades() + freqtradebot.enter_positions() msg = rpc._rpc_forcesell('all') assert msg == {'result': 'Created sell orders for all open trades.'} @@ -563,7 +620,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: assert cancel_order_mock.call_count == 1 assert trade.amount == filled_amount - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.filter(Trade.id == '2').first() amount = trade.amount # make an limit-buy open trade, if there is no 'filled', don't sell it @@ -582,7 +639,7 @@ def test_rpc_forcesell(default_conf, ticker, fee, mocker) -> None: assert cancel_order_mock.call_count == 2 assert trade.amount == amount - freqtradebot.create_trades() + freqtradebot.enter_positions() # make an limit-sell open trade mocker.patch( 'freqtrade.exchange.Exchange.get_order', @@ -604,7 +661,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -613,7 +670,7 @@ def test_performance_handle(default_conf, ticker, limit_buy_order, fee, rpc = RPC(freqtradebot) # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.first() assert trade @@ -637,7 +694,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -649,7 +706,7 @@ def test_rpc_count(mocker, default_conf, ticker, fee) -> None: assert counts["current"] == 0 # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() counts = rpc._rpc_count() assert counts["current"] == 1 @@ -661,7 +718,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, buy=buy_mm ) @@ -673,7 +730,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None trade = rpc._rpc_forcebuy(pair, None) assert isinstance(trade, Trade) assert trade.pair == pair - assert trade.open_rate == ticker()['ask'] + assert trade.open_rate == ticker()['bid'] # Test buy duplicate with pytest.raises(RPCException, match=r'position for ETH/BTC already open - id: 1'): @@ -686,7 +743,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order) -> None # Test buy pair not with stakes with pytest.raises(RPCException, match=r'Wrong pair selected. Please pairs with stake.*'): - rpc._rpc_forcebuy('XRP/ETH', 0.0001) + rpc._rpc_forcebuy('LTC/ETH', 0.0001) pair = 'XRP/BTC' # Test not buying diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index f1e3421c5..208a94c66 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -13,7 +13,7 @@ from freqtrade.__init__ import __version__ from freqtrade.persistence import Trade from freqtrade.rpc.api_server import BASE_URI, ApiServer from freqtrade.state import State -from tests.conftest import get_patched_freqtradebot, log_has, patch_get_signal +from tests.conftest import get_patched_freqtradebot, log_has, patch_get_signal, create_mock_trades _TEST_USER = "FreqTrader" _TEST_PASS = "SuperSecurePassword1!" @@ -49,6 +49,7 @@ def client_get(client, url): def assert_response(response, expected_code=200): assert response.status_code == expected_code assert response.content_type == "application/json" + assert ('Access-Control-Allow-Origin', '*') in response.headers._list def test_api_not_found(botclient): @@ -94,6 +95,33 @@ def test_api_unauthorized(botclient): assert rc.json == {'error': 'Unauthorized'} +def test_api_token_login(botclient): + ftbot, client = botclient + rc = client_post(client, f"{BASE_URI}/token/login") + assert_response(rc) + assert 'access_token' in rc.json + assert 'refresh_token' in rc.json + + # test Authentication is working with JWT tokens too + rc = client.get(f"{BASE_URI}/count", + content_type="application/json", + headers={'Authorization': f'Bearer {rc.json["access_token"]}'}) + assert_response(rc) + + +def test_api_token_refresh(botclient): + ftbot, client = botclient + rc = client_post(client, f"{BASE_URI}/token/login") + assert_response(rc) + rc = client.post(f"{BASE_URI}/token/refresh", + content_type="application/json", + data=None, + headers={'Authorization': f'Bearer {rc.json["refresh_token"]}'}) + assert_response(rc) + assert 'access_token' in rc.json + assert 'refresh_token' not in rc.json + + def test_api_stop_workflow(botclient): ftbot, client = botclient assert ftbot.state == State.RUNNING @@ -123,6 +151,12 @@ def test_api__init__(default_conf, mocker): """ Test __init__() method """ + default_conf.update({"api_server": {"enabled": True, + "listen_ip_address": "127.0.0.1", + "listen_port": 8080, + "username": "TestUser", + "password": "testPass", + }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) mocker.patch('freqtrade.rpc.api_server.ApiServer.run', MagicMock()) @@ -256,7 +290,7 @@ def test_api_count(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -267,7 +301,7 @@ def test_api_count(botclient, mocker, ticker, fee, markets): assert rc.json["max"] == 1.0 # Create some test data - ftbot.create_trades() + ftbot.enter_positions() rc = client_get(client, f"{BASE_URI}/count") assert_response(rc) assert rc.json["current"] == 1.0 @@ -283,6 +317,7 @@ def test_api_show_config(botclient, mocker): assert 'dry_run' in rc.json assert rc.json['exchange'] == 'bittrex' assert rc.json['ticker_interval'] == '5m' + assert rc.json['state'] == 'running' assert not rc.json['trailing_stop'] @@ -292,14 +327,40 @@ def test_api_daily(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) rc = client_get(client, f"{BASE_URI}/daily") assert_response(rc) - assert len(rc.json) == 7 - assert rc.json[0][0] == str(datetime.utcnow().date()) + assert len(rc.json['data']) == 7 + assert rc.json['stake_currency'] == 'BTC' + assert rc.json['fiat_display_currency'] == 'USD' + assert rc.json['data'][0]['date'] == str(datetime.utcnow().date()) + + +def test_api_trades(botclient, mocker, ticker, fee, markets): + ftbot, client = botclient + patch_get_signal(ftbot, (True, False)) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets) + ) + rc = client_get(client, f"{BASE_URI}/trades") + assert_response(rc) + assert len(rc.json) == 2 + assert rc.json['trades_count'] == 0 + + create_mock_trades(fee) + + rc = client_get(client, f"{BASE_URI}/trades") + assert_response(rc) + assert len(rc.json['trades']) == 3 + assert rc.json['trades_count'] == 3 + rc = client_get(client, f"{BASE_URI}/trades?limit=2") + assert_response(rc) + assert len(rc.json['trades']) == 2 + assert rc.json['trades_count'] == 2 def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): @@ -308,7 +369,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -323,7 +384,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -333,7 +394,7 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li assert len(rc.json) == 1 assert rc.json == {"error": "Error querying _profit: no closed trade"} - ftbot.create_trades() + ftbot.enter_positions() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade @@ -413,7 +474,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -422,29 +483,48 @@ def test_api_status(botclient, mocker, ticker, fee, markets): assert_response(rc, 200) assert rc.json == [] - ftbot.create_trades() + ftbot.enter_positions() rc = client_get(client, f"{BASE_URI}/status") assert_response(rc) assert len(rc.json) == 1 - assert rc.json == [{'amount': 90.99181074, + assert rc.json == [{'amount': 91.07468124, 'base_currency': 'BTC', 'close_date': None, 'close_date_hum': None, 'close_profit': None, 'close_rate': None, - 'current_profit': -0.59, - 'current_rate': 1.098e-05, + 'current_profit': -0.41, + 'current_rate': 1.099e-05, 'initial_stop_loss': 0.0, 'initial_stop_loss_pct': None, 'open_date': ANY, 'open_date_hum': 'just now', 'open_order': '(limit buy rem=0.00000000)', - 'open_rate': 1.099e-05, + 'open_rate': 1.098e-05, 'pair': 'ETH/BTC', 'stake_amount': 0.001, 'stop_loss': 0.0, 'stop_loss_pct': None, - 'trade_id': 1}] + 'trade_id': 1, + 'close_rate_requested': None, + 'current_rate': 1.099e-05, + 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, + 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, + 'open_date': ANY, + 'is_open': True, + 'max_rate': 0.0, + 'min_rate': None, + 'open_order_id': ANY, + 'open_rate_requested': 1.098e-05, + 'open_trade_price': 0.0010025, + 'sell_reason': None, + 'sell_order_status': None, + 'strategy': 'DefaultStrategy', + 'ticker_interval': 5}] def test_api_version(botclient): @@ -533,7 +613,26 @@ def test_api_forcebuy(botclient, mocker, fee): 'stake_amount': 1, 'stop_loss': None, 'stop_loss_pct': None, - 'trade_id': None} + 'trade_id': None, + 'close_profit': None, + 'close_rate_requested': None, + 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, + 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, + 'is_open': False, + 'max_rate': None, + 'min_rate': None, + 'open_order_id': '123456', + 'open_rate_requested': None, + 'open_trade_price': 0.2460546025, + 'sell_reason': None, + 'sell_order_status': None, + 'strategy': None, + 'ticker_interval': None + } def test_api_forcesell(botclient, mocker, ticker, fee, markets): @@ -541,7 +640,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value=ticker), - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, markets=PropertyMock(return_value=markets) ) @@ -552,7 +651,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): assert_response(rc, 502) assert rc.json == {"error": "Error querying _forcesell: invalid argument"} - ftbot.create_trades() + ftbot.enter_positions() rc = client_post(client, f"{BASE_URI}/forcesell", data='{"tradeid": "1"}') diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index b02f11394..b84073dcc 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -148,11 +148,6 @@ def test_status(default_conf, update, mocker, fee, ticker,) -> None: default_conf['telegram']['enabled'] = False default_conf['telegram']['chat_id'] = "123" - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_ticker=ticker, - get_fee=fee, - ) msg_mock = MagicMock() status_table = MagicMock() mocker.patch.multiple( @@ -175,6 +170,7 @@ def test_status(default_conf, update, mocker, fee, ticker,) -> None: 'current_profit': -0.59, 'initial_stop_loss': 1.098e-05, 'stop_loss': 1.099e-05, + 'sell_order_status': None, 'initial_stop_loss_pct': -0.05, 'stop_loss_pct': -0.01, 'open_order': '(limit buy rem=0.00000000)' @@ -184,13 +180,8 @@ def test_status(default_conf, update, mocker, fee, ticker,) -> None: ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) - patch_get_signal(freqtradebot, (True, False)) telegram = Telegram(freqtradebot) - # Create some test data - for _ in range(3): - freqtradebot.create_trades() - telegram._status(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -204,7 +195,7 @@ def test_status(default_conf, update, mocker, fee, ticker,) -> None: def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) msg_mock = MagicMock() @@ -236,7 +227,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: msg_mock.reset_mock() # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() # Trigger status while we have a fulfilled order for the open trade telegram._status(update=update, context=MagicMock()) @@ -254,7 +245,7 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) @@ -275,17 +266,17 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: # Status table is also enabled when stopped telegram._status_table(update=update, context=MagicMock()) assert msg_mock.call_count == 1 - assert 'no active order' in msg_mock.call_args_list[0][0][0] + assert 'no active trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() freqtradebot.state = State.RUNNING telegram._status_table(update=update, context=MagicMock()) assert msg_mock.call_count == 1 - assert 'no active order' in msg_mock.call_args_list[0][0][0] + assert 'no active trade' in msg_mock.call_args_list[0][0][0] msg_mock.reset_mock() # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() telegram._status_table(update=update, context=MagicMock()) @@ -294,7 +285,7 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: fields = re.sub('[ ]+', ' ', line[2].strip()).split(' ') assert int(fields[0]) == 1 - assert fields[1] == 'ETH/BTC' + assert 'ETH/BTC' in fields[1] assert msg_mock.call_count == 1 @@ -307,7 +298,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) msg_mock = MagicMock() @@ -322,7 +313,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.first() assert trade @@ -352,7 +343,8 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, msg_mock.reset_mock() freqtradebot.config['max_open_trades'] = 2 # Add two other trades - freqtradebot.create_trades() + n = freqtradebot.enter_positions() + assert n == 2 trades = Trade.query.all() for trade in trades: @@ -373,7 +365,7 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker + fetch_ticker=ticker ) msg_mock = MagicMock() mocker.patch.multiple( @@ -411,7 +403,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) msg_mock = MagicMock() @@ -431,7 +423,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, msg_mock.reset_mock() # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.first() # Simulate fulfilled LIMIT_BUY order for trade @@ -443,7 +435,7 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, msg_mock.reset_mock() # Update the ticker with a market going up - mocker.patch('freqtrade.exchange.Exchange.get_ticker', ticker_sell_up) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', ticker_sell_up) trade.update(limit_sell_order) trade.close_date = datetime.utcnow() @@ -534,7 +526,7 @@ def test_balance_handle_empty_response_dry(default_conf, update, mocker) -> None telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 - assert "*Warning:*Simulated balances in Dry Mode." in result + assert "*Warning:* Simulated balances in Dry Mode." in result assert "Starting capital: `1000` BTC" in result @@ -700,7 +692,7 @@ def test_forcesell_handle(default_conf, update, ticker, fee, patch_whitelist(mocker, default_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -709,13 +701,13 @@ def test_forcesell_handle(default_conf, update, ticker, fee, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.first() assert trade # Increase the price and sell it - mocker.patch('freqtrade.exchange.Exchange.get_ticker', ticker_sell_up) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', ticker_sell_up) # /forcesell 1 context = MagicMock() @@ -729,13 +721,13 @@ def test_forcesell_handle(default_conf, update, ticker, fee, 'exchange': 'Bittrex', 'pair': 'ETH/BTC', 'gain': 'profit', - 'limit': 1.172e-05, - 'amount': 90.99181073703367, + 'limit': 1.173e-05, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.172e-05, - 'profit_amount': 6.126e-05, - 'profit_percent': 0.0611052, + 'open_rate': 1.098e-05, + 'current_rate': 1.173e-05, + 'profit_amount': 6.314e-05, + 'profit_ratio': 0.0629778, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.FORCE_SELL.value, @@ -755,7 +747,7 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) @@ -764,12 +756,12 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_down + fetch_ticker=ticker_sell_down ) trade = Trade.query.first() @@ -788,13 +780,13 @@ def test_forcesell_down_handle(default_conf, update, ticker, fee, 'exchange': 'Bittrex', 'pair': 'ETH/BTC', 'gain': 'loss', - 'limit': 1.044e-05, - 'amount': 90.99181073703367, + 'limit': 1.043e-05, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.044e-05, - 'profit_amount': -5.492e-05, - 'profit_percent': -0.05478342, + 'open_rate': 1.098e-05, + 'current_rate': 1.043e-05, + 'profit_amount': -5.497e-05, + 'profit_ratio': -0.05482878, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.FORCE_SELL.value, @@ -812,7 +804,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None patch_whitelist(mocker, default_conf) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) default_conf['max_open_trades'] = 4 @@ -821,7 +813,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() rpc_mock.reset_mock() # /forcesell all @@ -836,13 +828,13 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None 'exchange': 'Bittrex', 'pair': 'ETH/BTC', 'gain': 'loss', - 'limit': 1.098e-05, - 'amount': 90.99181073703367, + 'limit': 1.099e-05, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.098e-05, - 'profit_amount': -5.91e-06, - 'profit_percent': -0.00589291, + 'open_rate': 1.098e-05, + 'current_rate': 1.099e-05, + 'profit_amount': -4.09e-06, + 'profit_ratio': -0.00408133, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.FORCE_SELL.value, @@ -963,7 +955,7 @@ def test_performance_handle(default_conf, update, ticker, fee, ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) freqtradebot = get_patched_freqtradebot(mocker, default_conf) @@ -971,7 +963,7 @@ def test_performance_handle(default_conf, update, ticker, fee, telegram = Telegram(freqtradebot) # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() trade = Trade.query.first() assert trade @@ -998,7 +990,7 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': 'mocked_order_id'}), get_fee=fee, ) @@ -1014,7 +1006,7 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: freqtradebot.state = State.RUNNING # Create some test data - freqtradebot.create_trades() + freqtradebot.enter_positions() msg_mock.reset_mock() telegram._count(update=update, context=MagicMock()) @@ -1209,12 +1201,35 @@ def test_send_msg_buy_notification(default_conf, mocker) -> None: 'stake_amount': 0.001, 'stake_amount_fiat': 0.0, 'stake_currency': 'BTC', - 'fiat_currency': 'USD' + 'fiat_currency': 'USD', + 'current_rate': 1.099e-05, + 'amount': 1333.3333333333335, + 'open_date': arrow.utcnow().shift(hours=-1) }) assert msg_mock.call_args[0][0] \ == '*Bittrex:* Buying ETH/BTC\n' \ - 'at rate `0.00001099\n' \ - '(0.001000 BTC,0.000 USD)`' + '*Amount:* `1333.33333333`\n' \ + '*Open Rate:* `0.00001099`\n' \ + '*Current Rate:* `0.00001099`\n' \ + '*Total:* `(0.001000 BTC, 12.345 USD)`' + + +def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + telegram.send_msg({ + 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + }) + assert msg_mock.call_args[0][0] \ + == ('*Bittrex:* Cancelling Open Buy Order for ETH/BTC') def test_send_msg_sell_notification(default_conf, mocker) -> None: @@ -1239,7 +1254,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'open_rate': 7.5e-05, 'current_rate': 3.201e-05, 'profit_amount': -0.05746268, - 'profit_percent': -0.57405275, + 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', 'fiat_currency': 'USD', 'sell_reason': SellType.STOP_LOSS.value, @@ -1248,13 +1263,13 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == ('*Binance:* Selling KEY/ETH\n' - '*Rate:* `0.00003201`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' + '*Close Rate:* `0.00003201`\n' '*Sell Reason:* `stop_loss`\n' '*Duration:* `1:00:00 (60.0 min)`\n' - '*Profit:* `-57.41%`` (loss: -0.05746268 ETH`` / -24.812 USD)`') + '*Profit:* `-57.41%` `(loss: -0.05746268 ETH / -24.812 USD)`') msg_mock.reset_mock() telegram.send_msg({ @@ -1268,7 +1283,7 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: 'open_rate': 7.5e-05, 'current_rate': 3.201e-05, 'profit_amount': -0.05746268, - 'profit_percent': -0.57405275, + 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', 'sell_reason': SellType.STOP_LOSS.value, 'open_date': arrow.utcnow().shift(days=-1, hours=-2, minutes=-30), @@ -1276,10 +1291,10 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == ('*Binance:* Selling KEY/ETH\n' - '*Rate:* `0.00003201`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' + '*Close Rate:* `0.00003201`\n' '*Sell Reason:* `stop_loss`\n' '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' '*Profit:* `-57.41%`') @@ -1287,6 +1302,39 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: telegram._fiat_converter.convert_amount = old_convamount +def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None: + msg_mock = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram = Telegram(freqtradebot) + old_convamount = telegram._fiat_converter.convert_amount + telegram._fiat_converter.convert_amount = lambda a, b, c: -24.812 + telegram.send_msg({ + 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'reason': 'Cancelled on exchange' + }) + assert msg_mock.call_args[0][0] \ + == ('*Binance:* Cancelling Open Sell Order for KEY/ETH. Reason: Cancelled on exchange') + + msg_mock.reset_mock() + telegram.send_msg({ + 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'exchange': 'Binance', + 'pair': 'KEY/ETH', + 'reason': 'timeout' + }) + assert msg_mock.call_args[0][0] \ + == ('*Binance:* Cancelling Open Sell Order for KEY/ETH. Reason: timeout') + # Reset singleton function to avoid random breaks + telegram._fiat_converter.convert_amount = old_convamount + + def test_send_msg_status_notification(default_conf, mocker) -> None: msg_mock = MagicMock() mocker.patch.multiple( @@ -1369,12 +1417,17 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None: 'stake_amount': 0.001, 'stake_amount_fiat': 0.0, 'stake_currency': 'BTC', - 'fiat_currency': None + 'fiat_currency': None, + 'current_rate': 1.099e-05, + 'amount': 1333.3333333333335, + 'open_date': arrow.utcnow().shift(hours=-1) }) assert msg_mock.call_args[0][0] \ == '*Bittrex:* Buying ETH/BTC\n' \ - 'at rate `0.00001099\n' \ - '(0.001000 BTC)`' + '*Amount:* `1333.33333333`\n' \ + '*Open Rate:* `0.00001099`\n' \ + '*Current Rate:* `0.00001099`\n' \ + '*Total:* `(0.001000 BTC)`' def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: @@ -1398,7 +1451,7 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: 'open_rate': 7.5e-05, 'current_rate': 3.201e-05, 'profit_amount': -0.05746268, - 'profit_percent': -0.57405275, + 'profit_ratio': -0.57405275, 'stake_currency': 'ETH', 'fiat_currency': 'USD', 'sell_reason': SellType.STOP_LOSS.value, @@ -1407,10 +1460,10 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == '*Binance:* Selling KEY/ETH\n' \ - '*Rate:* `0.00003201`\n' \ '*Amount:* `1333.33333333`\n' \ '*Open Rate:* `0.00007500`\n' \ '*Current Rate:* `0.00003201`\n' \ + '*Close Rate:* `0.00003201`\n' \ '*Sell Reason:* `stop_loss`\n' \ '*Duration:* `2:35:03 (155.1 min)`\n' \ '*Profit:* `-57.41%`' diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index c066aa8e7..1ced62746 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -13,24 +13,34 @@ from tests.conftest import get_patched_freqtradebot, log_has def get_webhook_dict() -> dict: return { - "enabled": True, - "url": "https://maker.ifttt.com/trigger/freqtrade_test/with/key/c764udvJ5jfSlswVRukZZ2/", - "webhookbuy": { - "value1": "Buying {pair}", - "value2": "limit {limit:8f}", - "value3": "{stake_amount:8f} {stake_currency}" - }, - "webhooksell": { - "value1": "Selling {pair}", - "value2": "limit {limit:8f}", - "value3": "profit: {profit_amount:8f} {stake_currency}" - }, - "webhookstatus": { - "value1": "Status: {status}", - "value2": "", - "value3": "" - } - } + "enabled": True, + "url": "https://maker.ifttt.com/trigger/freqtrade_test/with/key/c764udvJ5jfSlswVRukZZ2/", + "webhookbuy": { + "value1": "Buying {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, + "webhookbuycancel": { + "value1": "Cancelling Open Buy Order for {pair}", + "value2": "limit {limit:8f}", + "value3": "{stake_amount:8f} {stake_currency}" + }, + "webhooksell": { + "value1": "Selling {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})" + }, + "webhooksellcancel": { + "value1": "Cancelling Open Sell Order for {pair}", + "value2": "limit {limit:8f}", + "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})" + }, + "webhookstatus": { + "value1": "Status: {status}", + "value2": "", + "value3": "" + } + } def test__init__(mocker, default_conf): @@ -44,6 +54,9 @@ def test_send_msg(default_conf, mocker): msg_mock = MagicMock() mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) webhook = Webhook(get_patched_freqtradebot(mocker, default_conf)) + # Test buy + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) msg = { 'type': RPCMessageType.BUY_NOTIFICATION, 'exchange': 'Bittrex', @@ -54,8 +67,6 @@ def test_send_msg(default_conf, mocker): 'stake_currency': 'BTC', 'fiat_currency': 'EUR' } - msg_mock = MagicMock() - mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) webhook.send_msg(msg=msg) assert msg_mock.call_count == 1 assert (msg_mock.call_args[0][0]["value1"] == @@ -64,6 +75,27 @@ def test_send_msg(default_conf, mocker): default_conf["webhook"]["webhookbuy"]["value2"].format(**msg)) assert (msg_mock.call_args[0][0]["value3"] == default_conf["webhook"]["webhookbuy"]["value3"].format(**msg)) + # Test buy cancel + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + msg = { + 'type': RPCMessageType.BUY_CANCEL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'limit': 0.005, + 'stake_amount': 0.8, + 'stake_amount_fiat': 500, + 'stake_currency': 'BTC', + 'fiat_currency': 'EUR' + } + webhook.send_msg(msg=msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhookbuycancel"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhookbuycancel"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhookbuycancel"]["value3"].format(**msg)) # Test sell msg_mock = MagicMock() mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) @@ -78,7 +110,7 @@ def test_send_msg(default_conf, mocker): 'open_rate': 0.004, 'current_rate': 0.005, 'profit_amount': 0.001, - 'profit_percent': 0.20, + 'profit_ratio': 0.20, 'stake_currency': 'BTC', 'sell_reason': SellType.STOP_LOSS.value } @@ -90,7 +122,32 @@ def test_send_msg(default_conf, mocker): default_conf["webhook"]["webhooksell"]["value2"].format(**msg)) assert (msg_mock.call_args[0][0]["value3"] == default_conf["webhook"]["webhooksell"]["value3"].format(**msg)) - + # Test sell cancel + msg_mock = MagicMock() + mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock) + msg = { + 'type': RPCMessageType.SELL_CANCEL_NOTIFICATION, + 'exchange': 'Bittrex', + 'pair': 'ETH/BTC', + 'gain': "profit", + 'limit': 0.005, + 'amount': 0.8, + 'order_type': 'limit', + 'open_rate': 0.004, + 'current_rate': 0.005, + 'profit_amount': 0.001, + 'profit_ratio': 0.20, + 'stake_currency': 'BTC', + 'sell_reason': SellType.STOP_LOSS.value + } + webhook.send_msg(msg=msg) + assert msg_mock.call_count == 1 + assert (msg_mock.call_args[0][0]["value1"] == + default_conf["webhook"]["webhooksellcancel"]["value1"].format(**msg)) + assert (msg_mock.call_args[0][0]["value2"] == + default_conf["webhook"]["webhooksellcancel"]["value2"].format(**msg)) + assert (msg_mock.call_args[0][0]["value3"] == + default_conf["webhook"]["webhooksellcancel"]["value3"].format(**msg)) for msgtype in [RPCMessageType.STATUS_NOTIFICATION, RPCMessageType.WARNING_NOTIFICATION, RPCMessageType.CUSTOM_NOTIFICATION]: diff --git a/freqtrade/strategy/default_strategy.py b/tests/strategy/strats/default_strategy.py similarity index 98% rename from freqtrade/strategy/default_strategy.py rename to tests/strategy/strats/default_strategy.py index 6c343b477..7ea55d3f9 100644 --- a/freqtrade/strategy/default_strategy.py +++ b/tests/strategy/strats/default_strategy.py @@ -68,7 +68,7 @@ class DefaultStrategy(IStrategy): Performance Note: For the best performance be frugal on the number of indicators you are using. Let uncomment only the indicator you are using in your strategies or your hyperopt configuration, otherwise you will waste your memory and CPU usage. - :param dataframe: Raw data from the exchange and parsed by parse_ticker_dataframe() + :param dataframe: Dataframe with data from the exchange :param metadata: Additional information, like the currently traded pair :return: a Dataframe with all mandatory indicators for the strategies """ diff --git a/tests/strategy/strats/failing_strategy.py b/tests/strategy/strats/failing_strategy.py new file mode 100644 index 000000000..f8eaac3c3 --- /dev/null +++ b/tests/strategy/strats/failing_strategy.py @@ -0,0 +1,9 @@ +# The strategy which fails to load due to non-existent dependency + +import nonexiting_module # noqa + +from freqtrade.strategy.interface import IStrategy + + +class TestStrategyLegacy(IStrategy): + pass diff --git a/tests/strategy/legacy_strategy.py b/tests/strategy/strats/legacy_strategy.py similarity index 100% rename from tests/strategy/legacy_strategy.py rename to tests/strategy/strats/legacy_strategy.py diff --git a/tests/strategy/test_default_strategy.py b/tests/strategy/test_default_strategy.py index 17d6b8ee0..0b8ea9f85 100644 --- a/tests/strategy/test_default_strategy.py +++ b/tests/strategy/test_default_strategy.py @@ -1,6 +1,6 @@ from pandas import DataFrame -from freqtrade.strategy.default_strategy import DefaultStrategy +from .strats.default_strategy import DefaultStrategy def test_default_strategy_structure(): diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 605622b8f..dd6b11a06 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -4,94 +4,149 @@ import logging from unittest.mock import MagicMock import arrow +import pytest from pandas import DataFrame from freqtrade.configuration import TimeRange -from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.history import load_tickerdata_file +from freqtrade.data.history import load_data +from freqtrade.exceptions import StrategyError from freqtrade.persistence import Trade -from tests.conftest import get_patched_exchange, log_has -from freqtrade.strategy.default_strategy import DefaultStrategy +from freqtrade.resolvers import StrategyResolver +from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper +from tests.conftest import get_patched_exchange, log_has, log_has_re + +from .strats.default_strategy import DefaultStrategy # Avoid to reinit the same object again and again _STRATEGY = DefaultStrategy(config={}) -def test_returns_latest_buy_signal(mocker, default_conf, ticker_history): - mocker.patch.object( - _STRATEGY, '_analyze_ticker_internal', - return_value=DataFrame([{'buy': 1, 'sell': 0, 'date': arrow.utcnow()}]) - ) - assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False) +def test_returns_latest_signal(mocker, default_conf, ohlcv_history): + ohlcv_history.loc[1, 'date'] = arrow.utcnow() + # Take a copy to correctly modify the call + mocked_history = ohlcv_history.copy() + mocked_history['sell'] = 0 + mocked_history['buy'] = 0 + mocked_history.loc[1, 'sell'] = 1 mocker.patch.object( _STRATEGY, '_analyze_ticker_internal', - return_value=DataFrame([{'buy': 0, 'sell': 1, 'date': arrow.utcnow()}]) - ) - assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True) - - -def test_returns_latest_sell_signal(mocker, default_conf, ticker_history): - mocker.patch.object( - _STRATEGY, '_analyze_ticker_internal', - return_value=DataFrame([{'sell': 1, 'buy': 0, 'date': arrow.utcnow()}]) + return_value=mocked_history ) - assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (False, True) + assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (False, True) + mocked_history.loc[1, 'sell'] = 0 + mocked_history.loc[1, 'buy'] = 1 mocker.patch.object( _STRATEGY, '_analyze_ticker_internal', - return_value=DataFrame([{'sell': 0, 'buy': 1, 'date': arrow.utcnow()}]) + return_value=mocked_history ) - assert _STRATEGY.get_signal('ETH/BTC', '5m', ticker_history) == (True, False) + assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (True, False) + mocked_history.loc[1, 'sell'] = 0 + mocked_history.loc[1, 'buy'] = 0 + + mocker.patch.object( + _STRATEGY, '_analyze_ticker_internal', + return_value=mocked_history + ) + assert _STRATEGY.get_signal('ETH/BTC', '5m', ohlcv_history) == (False, False) def test_get_signal_empty(default_conf, mocker, caplog): assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], DataFrame()) - assert log_has('Empty ticker history for pair foo', caplog) + assert log_has('Empty candle (OHLCV) data for pair foo', caplog) caplog.clear() assert (False, False) == _STRATEGY.get_signal('bar', default_conf['ticker_interval'], []) - assert log_has('Empty ticker history for pair bar', caplog) + assert log_has('Empty candle (OHLCV) data for pair bar', caplog) -def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ticker_history): +def test_get_signal_exception_valueerror(default_conf, mocker, caplog, ohlcv_history): caplog.set_level(logging.INFO) mocker.patch.object( _STRATEGY, '_analyze_ticker_internal', side_effect=ValueError('xyz') ) assert (False, False) == _STRATEGY.get_signal('foo', default_conf['ticker_interval'], - ticker_history) - assert log_has('Unable to analyze ticker for pair foo: xyz', caplog) + ohlcv_history) + assert log_has_re(r'Strategy caused the following exception: xyz.*', caplog) -def test_get_signal_empty_dataframe(default_conf, mocker, caplog, ticker_history): +def test_get_signal_empty_dataframe(default_conf, mocker, caplog, ohlcv_history): caplog.set_level(logging.INFO) mocker.patch.object( _STRATEGY, '_analyze_ticker_internal', return_value=DataFrame([]) ) + mocker.patch.object(_STRATEGY, 'assert_df') + assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], - ticker_history) + ohlcv_history) assert log_has('Empty dataframe for pair xyz', caplog) -def test_get_signal_old_dataframe(default_conf, mocker, caplog, ticker_history): - caplog.set_level(logging.INFO) +def test_get_signal_old_dataframe(default_conf, mocker, caplog, ohlcv_history): # default_conf defines a 5m interval. we check interval * 2 + 5m # this is necessary as the last candle is removed (partial candles) by default - oldtime = arrow.utcnow().shift(minutes=-16) - ticks = DataFrame([{'buy': 1, 'date': oldtime}]) + ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16) + # Take a copy to correctly modify the call + mocked_history = ohlcv_history.copy() + mocked_history['sell'] = 0 + mocked_history['buy'] = 0 + mocked_history.loc[1, 'buy'] = 1 + + caplog.set_level(logging.INFO) mocker.patch.object( _STRATEGY, '_analyze_ticker_internal', - return_value=DataFrame(ticks) + return_value=mocked_history + ) + mocker.patch.object(_STRATEGY, 'assert_df') + assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], + ohlcv_history) + assert log_has('Outdated history for pair xyz. Last tick is 16 minutes old', caplog) + + +def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history): + # default_conf defines a 5m interval. we check interval * 2 + 5m + # this is necessary as the last candle is removed (partial candles) by default + ohlcv_history.loc[1, 'date'] = arrow.utcnow().shift(minutes=-16) + # Take a copy to correctly modify the call + mocked_history = ohlcv_history.copy() + mocked_history['sell'] = 0 + mocked_history['buy'] = 0 + mocked_history.loc[1, 'buy'] = 1 + + caplog.set_level(logging.INFO) + mocker.patch.object( + _STRATEGY, 'assert_df', + side_effect=StrategyError('Dataframe returned...') ) assert (False, False) == _STRATEGY.get_signal('xyz', default_conf['ticker_interval'], - ticker_history) - assert log_has('Outdated history for pair xyz. Last tick is 16 minutes old', caplog) + ohlcv_history) + assert log_has('Unable to analyze candle (OHLCV) data for pair xyz: Dataframe returned...', + caplog) + + +def test_assert_df(default_conf, mocker, ohlcv_history): + # Ensure it's running when passed correctly + _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), + ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date']) + + with pytest.raises(StrategyError, match=r"Dataframe returned from strategy.*length\."): + _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history) + 1, + ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date']) + + with pytest.raises(StrategyError, + match=r"Dataframe returned from strategy.*last close price\."): + _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), + ohlcv_history.loc[1, 'close'] + 0.01, ohlcv_history.loc[1, 'date']) + with pytest.raises(StrategyError, + match=r"Dataframe returned from strategy.*last date\."): + _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history), + ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date']) def test_get_signal_handles_exceptions(mocker, default_conf): @@ -103,15 +158,28 @@ def test_get_signal_handles_exceptions(mocker, default_conf): assert _STRATEGY.get_signal(exchange, 'ETH/BTC', '5m') == (False, False) -def test_tickerdata_to_dataframe(default_conf, testdatadir) -> None: - strategy = DefaultStrategy(default_conf) +def test_ohlcvdata_to_dataframe(default_conf, testdatadir) -> None: + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) timerange = TimeRange.parse_timerange('1510694220-1510700340') - tick = load_tickerdata_file(testdatadir, 'UNITTEST/BTC', '1m', timerange=timerange) - tickerlist = {'UNITTEST/BTC': parse_ticker_dataframe(tick, '1m', pair="UNITTEST/BTC", - fill_missing=True)} - data = strategy.tickerdata_to_dataframe(tickerlist) - assert len(data['UNITTEST/BTC']) == 102 # partial candle was removed + data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, + fill_up_missing=True) + processed = strategy.ohlcvdata_to_dataframe(data) + assert len(processed['UNITTEST/BTC']) == 102 # partial candle was removed + + +def test_ohlcvdata_to_dataframe_copy(mocker, default_conf, testdatadir) -> None: + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) + aimock = mocker.patch('freqtrade.strategy.interface.IStrategy.advise_indicators') + timerange = TimeRange.parse_timerange('1510694220-1510700340') + data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], timerange=timerange, + fill_up_missing=True) + strategy.ohlcvdata_to_dataframe(data) + assert aimock.call_count == 1 + # Ensure that a copy of the dataframe is passed to advice_indicators + assert aimock.call_args_list[0][0][0] is not data def test_min_roi_reached(default_conf, fee) -> None: @@ -120,7 +188,8 @@ def test_min_roi_reached(default_conf, fee) -> None: min_roi_list = [{20: 0.05, 55: 0.01, 0: 0.1}, {0: 0.1, 20: 0.05, 55: 0.01}] for roi in min_roi_list: - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) strategy.minimal_roi = roi trade = Trade( pair='ETH/BTC', @@ -158,7 +227,8 @@ def test_min_roi_reached2(default_conf, fee) -> None: }, ] for roi in min_roi_list: - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) strategy.minimal_roi = roi trade = Trade( pair='ETH/BTC', @@ -192,7 +262,8 @@ def test_min_roi_reached3(default_conf, fee) -> None: 30: 0.05, 55: 0.30, } - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) strategy.minimal_roi = min_roi trade = Trade( pair='ETH/BTC', @@ -219,7 +290,7 @@ def test_min_roi_reached3(default_conf, fee) -> None: assert strategy.min_roi_reached(trade, 0.31, arrow.utcnow().shift(minutes=-2).datetime) -def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: +def test_analyze_ticker_default(ohlcv_history, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) buy_mock = MagicMock(side_effect=lambda x, meta: x) @@ -232,7 +303,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: ) strategy = DefaultStrategy({}) - strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + strategy.analyze_ticker(ohlcv_history, {'pair': 'ETH/BTC'}) assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 assert buy_mock.call_count == 1 @@ -241,7 +312,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: assert not log_has('Skipping TA Analysis for already analyzed candle', caplog) caplog.clear() - strategy.analyze_ticker(ticker_history, {'pair': 'ETH/BTC'}) + strategy.analyze_ticker(ohlcv_history, {'pair': 'ETH/BTC'}) # No analysis happens as process_only_new_candles is true assert ind_mock.call_count == 2 assert buy_mock.call_count == 2 @@ -250,7 +321,7 @@ def test_analyze_ticker_default(ticker_history, mocker, caplog) -> None: assert not log_has('Skipping TA Analysis for already analyzed candle', caplog) -def test__analyze_ticker_internal_skip_analyze(ticker_history, mocker, caplog) -> None: +def test__analyze_ticker_internal_skip_analyze(ohlcv_history, mocker, caplog) -> None: caplog.set_level(logging.DEBUG) ind_mock = MagicMock(side_effect=lambda x, meta: x) buy_mock = MagicMock(side_effect=lambda x, meta: x) @@ -265,7 +336,7 @@ def test__analyze_ticker_internal_skip_analyze(ticker_history, mocker, caplog) - strategy = DefaultStrategy({}) strategy.process_only_new_candles = True - ret = strategy._analyze_ticker_internal(ticker_history, {'pair': 'ETH/BTC'}) + ret = strategy._analyze_ticker_internal(ohlcv_history, {'pair': 'ETH/BTC'}) assert 'high' in ret.columns assert 'low' in ret.columns assert 'close' in ret.columns @@ -277,7 +348,7 @@ def test__analyze_ticker_internal_skip_analyze(ticker_history, mocker, caplog) - assert not log_has('Skipping TA Analysis for already analyzed candle', caplog) caplog.clear() - ret = strategy._analyze_ticker_internal(ticker_history, {'pair': 'ETH/BTC'}) + ret = strategy._analyze_ticker_internal(ohlcv_history, {'pair': 'ETH/BTC'}) # No analysis happens as process_only_new_candles is true assert ind_mock.call_count == 1 assert buy_mock.call_count == 1 @@ -292,7 +363,8 @@ def test__analyze_ticker_internal_skip_analyze(ticker_history, mocker, caplog) - def test_is_pair_locked(default_conf): - strategy = DefaultStrategy(default_conf) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) # dict should be empty assert not strategy._pair_locked_until @@ -302,6 +374,54 @@ def test_is_pair_locked(default_conf): # ETH/BTC locked for 4 minutes assert strategy.is_pair_locked(pair) + # Test lock does not change + lock = strategy._pair_locked_until[pair] + strategy.lock_pair(pair, arrow.utcnow().shift(minutes=2).datetime) + assert lock == strategy._pair_locked_until[pair] + # XRP/BTC should not be locked now pair = 'XRP/BTC' assert not strategy.is_pair_locked(pair) + + # Unlocking a pair that's not locked should not raise an error + strategy.unlock_pair(pair) + + # Unlock original pair + pair = 'ETH/BTC' + strategy.unlock_pair(pair) + assert not strategy.is_pair_locked(pair) + + +@pytest.mark.parametrize('error', [ + ValueError, KeyError, Exception, +]) +def test_strategy_safe_wrapper_error(caplog, error): + def failing_method(): + raise error('This is an error.') + + def working_method(argumentpassedin): + return argumentpassedin + + with pytest.raises(StrategyError, match=r'This is an error.'): + strategy_safe_wrapper(failing_method, message='DeadBeef')() + + assert log_has_re(r'DeadBeef.*', caplog) + ret = strategy_safe_wrapper(failing_method, message='DeadBeef', default_retval=True)() + + assert isinstance(ret, bool) + assert ret + + +@pytest.mark.parametrize('value', [ + 1, 22, 55, True, False, {'a': 1, 'b': '112'}, + [1, 2, 3, 4], (4, 2, 3, 6) +]) +def test_strategy_safe_wrapper(value): + + def working_method(argumentpassedin): + return argumentpassedin + + ret = strategy_safe_wrapper(working_method, message='DeadBeef')(value) + + assert type(ret) == type(value) + assert ret == value diff --git a/tests/strategy/test_strategy.py b/tests/strategy/test_strategy.py index 963d36c76..13ca68bf0 100644 --- a/tests/strategy/test_strategy.py +++ b/tests/strategy/test_strategy.py @@ -2,45 +2,58 @@ import logging import warnings from base64 import urlsafe_b64encode -from os import path from pathlib import Path import pytest from pandas import DataFrame -from freqtrade import OperationalException +from freqtrade.exceptions import OperationalException from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.interface import IStrategy from tests.conftest import log_has, log_has_re def test_search_strategy(): - default_config = {} - default_location = Path(__file__).parent.parent.joinpath('strategy').resolve() + default_location = Path(__file__).parent / 'strats' s, _ = StrategyResolver._search_object( directory=default_location, - object_type=IStrategy, - kwargs={'config': default_config}, object_name='DefaultStrategy' ) - assert isinstance(s, IStrategy) + assert issubclass(s, IStrategy) s, _ = StrategyResolver._search_object( directory=default_location, - object_type=IStrategy, - kwargs={'config': default_config}, object_name='NotFoundStrategy' ) assert s is None +def test_search_all_strategies_no_failed(): + directory = Path(__file__).parent / "strats" + strategies = StrategyResolver.search_all_objects(directory, enum_failed=False) + assert isinstance(strategies, list) + assert len(strategies) == 2 + assert isinstance(strategies[0], dict) + + +def test_search_all_strategies_with_failed(): + directory = Path(__file__).parent / "strats" + strategies = StrategyResolver.search_all_objects(directory, enum_failed=True) + assert isinstance(strategies, list) + assert len(strategies) == 3 + # with enum_failed=True search_all_objects() shall find 2 good strategies + # and 1 which fails to load + assert len([x for x in strategies if x['class'] is not None]) == 2 + assert len([x for x in strategies if x['class'] is None]) == 1 + + def test_load_strategy(default_conf, result): default_conf.update({'strategy': 'SampleStrategy', 'strategy_path': str(Path(__file__).parents[2] / 'freqtrade/templates') }) - resolver = StrategyResolver(default_conf) - assert 'rsi' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) + strategy = StrategyResolver.load_strategy(default_conf) + assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) def test_load_strategy_base64(result, caplog, default_conf): @@ -48,8 +61,8 @@ def test_load_strategy_base64(result, caplog, default_conf): encoded_string = urlsafe_b64encode(file.read()).decode("utf-8") default_conf.update({'strategy': 'SampleStrategy:{}'.format(encoded_string)}) - resolver = StrategyResolver(default_conf) - assert 'rsi' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) + strategy = StrategyResolver.load_strategy(default_conf) + assert 'rsi' in strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) # Make sure strategy was loaded from base64 (using temp directory)!! assert log_has_re(r"Using resolved strategy SampleStrategy from '" r".*(/|\\).*(/|\\)SampleStrategy\.py'\.\.\.", caplog) @@ -57,21 +70,20 @@ def test_load_strategy_base64(result, caplog, default_conf): def test_load_strategy_invalid_directory(result, caplog, default_conf): default_conf['strategy'] = 'DefaultStrategy' - resolver = StrategyResolver(default_conf) extra_dir = Path.cwd() / 'some/path' - resolver._load_strategy('DefaultStrategy', config=default_conf, extra_dir=extra_dir) + with pytest.raises(OperationalException): + StrategyResolver._load_strategy('DefaultStrategy', config=default_conf, + extra_dir=extra_dir) assert log_has_re(r'Path .*' + r'some.*path.*' + r'.* does not exist', caplog) - assert 'rsi' in resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) - def test_load_not_found_strategy(default_conf): default_conf['strategy'] = 'NotFoundStrategy' with pytest.raises(OperationalException, match=r"Impossible to load Strategy 'NotFoundStrategy'. " r"This class does not exist or contains Python code errors."): - StrategyResolver(default_conf) + StrategyResolver.load_strategy(default_conf) def test_load_strategy_noname(default_conf): @@ -79,30 +91,30 @@ def test_load_strategy_noname(default_conf): with pytest.raises(OperationalException, match="No strategy set. Please use `--strategy` to specify " "the strategy class to use."): - StrategyResolver(default_conf) + StrategyResolver.load_strategy(default_conf) def test_strategy(result, default_conf): default_conf.update({'strategy': 'DefaultStrategy'}) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) metadata = {'pair': 'ETH/BTC'} - assert resolver.strategy.minimal_roi[0] == 0.04 + assert strategy.minimal_roi[0] == 0.04 assert default_conf["minimal_roi"]['0'] == 0.04 - assert resolver.strategy.stoploss == -0.10 + assert strategy.stoploss == -0.10 assert default_conf['stoploss'] == -0.10 - assert resolver.strategy.ticker_interval == '5m' + assert strategy.ticker_interval == '5m' assert default_conf['ticker_interval'] == '5m' - df_indicators = resolver.strategy.advise_indicators(result, metadata=metadata) + df_indicators = strategy.advise_indicators(result, metadata=metadata) assert 'adx' in df_indicators - dataframe = resolver.strategy.advise_buy(df_indicators, metadata=metadata) + dataframe = strategy.advise_buy(df_indicators, metadata=metadata) assert 'buy' in dataframe.columns - dataframe = resolver.strategy.advise_sell(df_indicators, metadata=metadata) + dataframe = strategy.advise_sell(df_indicators, metadata=metadata) assert 'sell' in dataframe.columns @@ -114,9 +126,9 @@ def test_strategy_override_minimal_roi(caplog, default_conf): "0": 0.5 } }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.minimal_roi[0] == 0.5 + assert strategy.minimal_roi[0] == 0.5 assert log_has("Override strategy 'minimal_roi' with value in config file: {'0': 0.5}.", caplog) @@ -126,9 +138,9 @@ def test_strategy_override_stoploss(caplog, default_conf): 'strategy': 'DefaultStrategy', 'stoploss': -0.5 }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.stoploss == -0.5 + assert strategy.stoploss == -0.5 assert log_has("Override strategy 'stoploss' with value in config file: -0.5.", caplog) @@ -138,10 +150,10 @@ def test_strategy_override_trailing_stop(caplog, default_conf): 'strategy': 'DefaultStrategy', 'trailing_stop': True }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.trailing_stop - assert isinstance(resolver.strategy.trailing_stop, bool) + assert strategy.trailing_stop + assert isinstance(strategy.trailing_stop, bool) assert log_has("Override strategy 'trailing_stop' with value in config file: True.", caplog) @@ -153,13 +165,13 @@ def test_strategy_override_trailing_stop_positive(caplog, default_conf): 'trailing_stop_positive_offset': -0.2 }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.trailing_stop_positive == -0.1 + assert strategy.trailing_stop_positive == -0.1 assert log_has("Override strategy 'trailing_stop_positive' with value in config file: -0.1.", caplog) - assert resolver.strategy.trailing_stop_positive_offset == -0.2 + assert strategy.trailing_stop_positive_offset == -0.2 assert log_has("Override strategy 'trailing_stop_positive' with value in config file: -0.1.", caplog) @@ -172,10 +184,10 @@ def test_strategy_override_ticker_interval(caplog, default_conf): 'ticker_interval': 60, 'stake_currency': 'ETH' }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.ticker_interval == 60 - assert resolver.strategy.stake_currency == 'ETH' + assert strategy.ticker_interval == 60 + assert strategy.stake_currency == 'ETH' assert log_has("Override strategy 'ticker_interval' with value in config file: 60.", caplog) @@ -187,9 +199,9 @@ def test_strategy_override_process_only_new_candles(caplog, default_conf): 'strategy': 'DefaultStrategy', 'process_only_new_candles': True }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.process_only_new_candles + assert strategy.process_only_new_candles assert log_has("Override strategy 'process_only_new_candles' with value in config file: True.", caplog) @@ -207,11 +219,11 @@ def test_strategy_override_order_types(caplog, default_conf): 'strategy': 'DefaultStrategy', 'order_types': order_types }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.order_types + assert strategy.order_types for method in ['buy', 'sell', 'stoploss', 'stoploss_on_exchange']: - assert resolver.strategy.order_types[method] == order_types[method] + assert strategy.order_types[method] == order_types[method] assert log_has("Override strategy 'order_types' with value in config file:" " {'buy': 'market', 'sell': 'limit', 'stoploss': 'limit'," @@ -225,7 +237,7 @@ def test_strategy_override_order_types(caplog, default_conf): with pytest.raises(ImportError, match=r"Impossible to load Strategy 'DefaultStrategy'. " r"Order-types mapping is incomplete."): - StrategyResolver(default_conf) + StrategyResolver.load_strategy(default_conf) def test_strategy_override_order_tif(caplog, default_conf): @@ -240,11 +252,11 @@ def test_strategy_override_order_tif(caplog, default_conf): 'strategy': 'DefaultStrategy', 'order_time_in_force': order_time_in_force }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.order_time_in_force + assert strategy.order_time_in_force for method in ['buy', 'sell']: - assert resolver.strategy.order_time_in_force[method] == order_time_in_force[method] + assert strategy.order_time_in_force[method] == order_time_in_force[method] assert log_has("Override strategy 'order_time_in_force' with value in config file:" " {'buy': 'fok', 'sell': 'gtc'}.", caplog) @@ -257,7 +269,7 @@ def test_strategy_override_order_tif(caplog, default_conf): with pytest.raises(ImportError, match=r"Impossible to load Strategy 'DefaultStrategy'. " r"Order-time-in-force mapping is incomplete."): - StrategyResolver(default_conf) + StrategyResolver.load_strategy(default_conf) def test_strategy_override_use_sell_signal(caplog, default_conf): @@ -265,9 +277,9 @@ def test_strategy_override_use_sell_signal(caplog, default_conf): default_conf.update({ 'strategy': 'DefaultStrategy', }) - resolver = StrategyResolver(default_conf) - assert resolver.strategy.use_sell_signal - assert isinstance(resolver.strategy.use_sell_signal, bool) + strategy = StrategyResolver.load_strategy(default_conf) + assert strategy.use_sell_signal + assert isinstance(strategy.use_sell_signal, bool) # must be inserted to configuration assert 'use_sell_signal' in default_conf['ask_strategy'] assert default_conf['ask_strategy']['use_sell_signal'] @@ -278,10 +290,10 @@ def test_strategy_override_use_sell_signal(caplog, default_conf): 'use_sell_signal': False, }, }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert not resolver.strategy.use_sell_signal - assert isinstance(resolver.strategy.use_sell_signal, bool) + assert not strategy.use_sell_signal + assert isinstance(strategy.use_sell_signal, bool) assert log_has("Override strategy 'use_sell_signal' with value in config file: False.", caplog) @@ -290,9 +302,9 @@ def test_strategy_override_use_sell_profit_only(caplog, default_conf): default_conf.update({ 'strategy': 'DefaultStrategy', }) - resolver = StrategyResolver(default_conf) - assert not resolver.strategy.sell_profit_only - assert isinstance(resolver.strategy.sell_profit_only, bool) + strategy = StrategyResolver.load_strategy(default_conf) + assert not strategy.sell_profit_only + assert isinstance(strategy.sell_profit_only, bool) # must be inserted to configuration assert 'sell_profit_only' in default_conf['ask_strategy'] assert not default_conf['ask_strategy']['sell_profit_only'] @@ -303,23 +315,23 @@ def test_strategy_override_use_sell_profit_only(caplog, default_conf): 'sell_profit_only': True, }, }) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) - assert resolver.strategy.sell_profit_only - assert isinstance(resolver.strategy.sell_profit_only, bool) + assert strategy.sell_profit_only + assert isinstance(strategy.sell_profit_only, bool) assert log_has("Override strategy 'sell_profit_only' with value in config file: True.", caplog) @pytest.mark.filterwarnings("ignore:deprecated") def test_deprecate_populate_indicators(result, default_conf): - default_location = path.join(path.dirname(path.realpath(__file__))) + default_location = Path(__file__).parent / "strats" default_conf.update({'strategy': 'TestStrategyLegacy', 'strategy_path': default_location}) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - indicators = resolver.strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) + indicators = strategy.advise_indicators(result, {'pair': 'ETH/BTC'}) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - check out the Sample strategy to see the current function headers!" \ @@ -328,7 +340,7 @@ def test_deprecate_populate_indicators(result, default_conf): with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - resolver.strategy.advise_buy(indicators, {'pair': 'ETH/BTC'}) + strategy.advise_buy(indicators, {'pair': 'ETH/BTC'}) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - check out the Sample strategy to see the current function headers!" \ @@ -337,7 +349,7 @@ def test_deprecate_populate_indicators(result, default_conf): with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") - resolver.strategy.advise_sell(indicators, {'pair': 'ETH_BTC'}) + strategy.advise_sell(indicators, {'pair': 'ETH_BTC'}) assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) assert "deprecated - check out the Sample strategy to see the current function headers!" \ @@ -346,50 +358,50 @@ def test_deprecate_populate_indicators(result, default_conf): @pytest.mark.filterwarnings("ignore:deprecated") def test_call_deprecated_function(result, monkeypatch, default_conf): - default_location = path.join(path.dirname(path.realpath(__file__))) + default_location = Path(__file__).parent / "strats" default_conf.update({'strategy': 'TestStrategyLegacy', 'strategy_path': default_location}) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) metadata = {'pair': 'ETH/BTC'} # Make sure we are using a legacy function - assert resolver.strategy._populate_fun_len == 2 - assert resolver.strategy._buy_fun_len == 2 - assert resolver.strategy._sell_fun_len == 2 - assert resolver.strategy.INTERFACE_VERSION == 1 + assert strategy._populate_fun_len == 2 + assert strategy._buy_fun_len == 2 + assert strategy._sell_fun_len == 2 + assert strategy.INTERFACE_VERSION == 1 - indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) + indicator_df = strategy.advise_indicators(result, metadata=metadata) assert isinstance(indicator_df, DataFrame) assert 'adx' in indicator_df.columns - buydf = resolver.strategy.advise_buy(result, metadata=metadata) + buydf = strategy.advise_buy(result, metadata=metadata) assert isinstance(buydf, DataFrame) assert 'buy' in buydf.columns - selldf = resolver.strategy.advise_sell(result, metadata=metadata) + selldf = strategy.advise_sell(result, metadata=metadata) assert isinstance(selldf, DataFrame) assert 'sell' in selldf def test_strategy_interface_versioning(result, monkeypatch, default_conf): default_conf.update({'strategy': 'DefaultStrategy'}) - resolver = StrategyResolver(default_conf) + strategy = StrategyResolver.load_strategy(default_conf) metadata = {'pair': 'ETH/BTC'} # Make sure we are using a legacy function - assert resolver.strategy._populate_fun_len == 3 - assert resolver.strategy._buy_fun_len == 3 - assert resolver.strategy._sell_fun_len == 3 - assert resolver.strategy.INTERFACE_VERSION == 2 + assert strategy._populate_fun_len == 3 + assert strategy._buy_fun_len == 3 + assert strategy._sell_fun_len == 3 + assert strategy.INTERFACE_VERSION == 2 - indicator_df = resolver.strategy.advise_indicators(result, metadata=metadata) + indicator_df = strategy.advise_indicators(result, metadata=metadata) assert isinstance(indicator_df, DataFrame) assert 'adx' in indicator_df.columns - buydf = resolver.strategy.advise_buy(result, metadata=metadata) + buydf = strategy.advise_buy(result, metadata=metadata) assert isinstance(buydf, DataFrame) assert 'buy' in buydf.columns - selldf = resolver.strategy.advise_sell(result, metadata=metadata) + selldf = strategy.advise_sell(result, metadata=metadata) assert isinstance(selldf, DataFrame) assert 'sell' in selldf diff --git a/tests/test_arguments.py b/tests/test_arguments.py index d8fbace0f..0052a61d0 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock import pytest -from freqtrade.configuration import Arguments -from freqtrade.configuration.cli_options import check_int_positive +from freqtrade.commands import Arguments +from freqtrade.commands.cli_options import check_int_positive # Parse common command-line-arguments. Used for all tools @@ -18,7 +18,8 @@ def test_parse_args_none() -> None: assert isinstance(arguments.parser, argparse.ArgumentParser) -def test_parse_args_defaults() -> None: +def test_parse_args_defaults(mocker) -> None: + mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True])) args = Arguments(['trade']).get_parsed_arg() assert args["config"] == ['config.json'] assert args["strategy_path"] is None @@ -26,6 +27,26 @@ def test_parse_args_defaults() -> None: assert args["verbosity"] == 0 +def test_parse_args_default_userdatadir(mocker) -> None: + mocker.patch.object(Path, "is_file", MagicMock(return_value=True)) + args = Arguments(['trade']).get_parsed_arg() + # configuration defaults to user_data if that is available. + assert args["config"] == [str(Path('user_data/config.json'))] + assert args["strategy_path"] is None + assert args["datadir"] is None + assert args["verbosity"] == 0 + + +def test_parse_args_userdatadir(mocker) -> None: + mocker.patch.object(Path, "is_file", MagicMock(return_value=True)) + args = Arguments(['trade', '--user-data-dir', 'user_data']).get_parsed_arg() + # configuration defaults to user_data if that is available. + assert args["config"] == [str(Path('user_data/config.json'))] + assert args["strategy_path"] is None + assert args["datadir"] is None + assert args["verbosity"] == 0 + + def test_parse_args_config() -> None: args = Arguments(['trade', '-c', '/dev/null']).get_parsed_arg() assert args["config"] == ['/dev/null'] @@ -208,7 +229,7 @@ def test_config_notrequired(mocker) -> None: assert pargs["config"] is None # When file exists: - mocker.patch.object(Path, "is_file", MagicMock(return_value=True)) + mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True])) args = [ 'download-data', ] diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 292d53315..edcbe4516 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -10,16 +10,17 @@ from unittest.mock import MagicMock import pytest from jsonschema import ValidationError -from freqtrade import OperationalException -from freqtrade.configuration import (Arguments, Configuration, check_exchange, +from freqtrade.commands import Arguments +from freqtrade.configuration import (Configuration, check_exchange, remove_credentials, validate_config_consistency) from freqtrade.configuration.config_validation import validate_config_schema from freqtrade.configuration.deprecated_settings import ( check_conflicting_settings, process_deprecated_setting, process_temporary_deprecated_settings) -from freqtrade.configuration.load_config import load_config_file +from freqtrade.configuration.load_config import load_config_file, log_config_error_range from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL +from freqtrade.exceptions import OperationalException from freqtrade.loggers import _set_loggers, setup_logging from freqtrade.state import RunMode from tests.conftest import (log_has, log_has_re, @@ -33,13 +34,6 @@ def all_conf(): return conf -def test_load_config_invalid_pair(default_conf) -> None: - default_conf['exchange']['pair_whitelist'].append('ETH-BTC') - - with pytest.raises(ValidationError, match=r'.*does not match.*'): - validate_config_schema(default_conf) - - def test_load_config_missing_attributes(default_conf) -> None: conf = deepcopy(default_conf) conf.pop('exchange') @@ -49,6 +43,7 @@ def test_load_config_missing_attributes(default_conf) -> None: conf = deepcopy(default_conf) conf.pop('stake_currency') + conf['runmode'] = RunMode.DRY_RUN with pytest.raises(ValidationError, match=r".*'stake_currency' is a required property.*"): validate_config_schema(conf) @@ -71,6 +66,30 @@ def test_load_config_file(default_conf, mocker, caplog) -> None: assert validated_conf.items() >= default_conf.items() +def test_load_config_file_error(default_conf, mocker, caplog) -> None: + del default_conf['user_data_dir'] + filedata = json.dumps(default_conf).replace( + '"stake_amount": 0.001,', '"stake_amount": .001,') + mocker.patch('freqtrade.configuration.load_config.open', mocker.mock_open(read_data=filedata)) + mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata)) + + with pytest.raises(OperationalException, match=r".*Please verify the following segment.*"): + load_config_file('somefile') + + +def test_load_config_file_error_range(default_conf, mocker, caplog) -> None: + del default_conf['user_data_dir'] + filedata = json.dumps(default_conf).replace( + '"stake_amount": 0.001,', '"stake_amount": .001,') + mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata)) + + x = log_config_error_range('somefile', 'Parse error at offset 64: Invalid value.') + assert isinstance(x, str) + assert (x == '{"max_open_trades": 1, "stake_currency": "BTC", ' + '"stake_amount": .001, "fiat_display_currency": "USD", ' + '"ticker_interval": "5m", "dry_run": true, ') + + def test__args_to_config(caplog): arg_list = ['trade', '--strategy-path', 'TestTest'] @@ -78,6 +97,7 @@ def test__args_to_config(caplog): configuration = Configuration(args) config = {} with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") # No warnings ... configuration._args_to_config(config, argname="strategy_path", logstring="DeadBeef") assert len(w) == 0 @@ -87,6 +107,7 @@ def test__args_to_config(caplog): configuration = Configuration(args) config = {} with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") # Deprecation warnings! configuration._args_to_config(config, argname="strategy_path", logstring="DeadBeef", deprecated_msg="Going away soon!") @@ -210,6 +231,7 @@ def test_load_config_file_exception(mocker) -> None: def test_load_config(default_conf, mocker) -> None: + del default_conf['strategy_path'] patched_configuration_load_config_file(mocker, default_conf) args = Arguments(['trade']).get_parsed_arg() @@ -322,7 +344,8 @@ def test_load_dry_run(default_conf, mocker, config_value, expected, arglist) -> configuration = Configuration(Arguments(arglist).get_parsed_arg()) validated_conf = configuration.load_config() - assert validated_conf.get('dry_run') is expected + assert validated_conf['dry_run'] is expected + assert validated_conf['runmode'] == (RunMode.DRY_RUN if expected else RunMode.LIVE) def test_load_custom_strategy(default_conf, mocker) -> None: @@ -722,6 +745,14 @@ def test_validate_default_conf(default_conf) -> None: validate_config_schema(default_conf) +def test_validate_max_open_trades(default_conf): + default_conf['max_open_trades'] = float('inf') + default_conf['stake_amount'] = 'unlimited' + with pytest.raises(OperationalException, match='`max_open_trades` and `stake_amount` ' + 'cannot both be unlimited.'): + validate_config_consistency(default_conf) + + def test_validate_tsl(default_conf): default_conf['stoploss'] = 0.0 with pytest.raises(OperationalException, match='The config stoploss needs to be different ' @@ -799,12 +830,6 @@ def test_validate_whitelist(default_conf): validate_config_consistency(conf) - conf = deepcopy(default_conf) - conf['stake_currency'] = 'USDT' - with pytest.raises(OperationalException, - match=r"Stake-currency 'USDT' not compatible with pair-whitelist.*"): - validate_config_consistency(conf) - def test_load_config_test_comments() -> None: """ @@ -977,7 +1002,7 @@ def test_pairlist_resolving_fallback(mocker): assert config['pairs'] == ['ETH/BTC', 'XRP/BTC'] assert config['exchange']['name'] == 'binance' - assert config['datadir'] == str(Path.cwd() / "user_data/data/binance") + assert config['datadir'] == Path.cwd() / "user_data/data/binance" @pytest.mark.parametrize("setting", [ @@ -1016,16 +1041,15 @@ def test_process_temporary_deprecated_settings(mocker, default_conf, setting, ca assert default_conf[setting[0]][setting[1]] == setting[5] -def test_process_deprecated_setting_pairlists(mocker, default_conf, caplog): - patched_configuration_load_config_file(mocker, default_conf) - default_conf.update({'pairlist': { - 'method': 'VolumePairList', - 'config': {'precision_filter': True} +def test_process_deprecated_setting_edge(mocker, edge_conf, caplog): + patched_configuration_load_config_file(mocker, edge_conf) + edge_conf.update({'edge': { + 'enabled': True, + 'capital_available_percentage': 0.5, }}) - process_temporary_deprecated_settings(default_conf) - assert log_has_re(r'DEPRECATED.*precision_filter.*', caplog) - assert log_has_re(r'DEPRECATED.*in pairlist is deprecated and must be moved*', caplog) + process_temporary_deprecated_settings(edge_conf) + assert log_has_re(r"DEPRECATED.*Using 'edge.capital_available_percentage'*", caplog) def test_check_conflicting_settings(mocker, default_conf, caplog): diff --git a/tests/test_directory_operations.py b/tests/test_directory_operations.py index db41e2da2..71c91549f 100644 --- a/tests/test_directory_operations.py +++ b/tests/test_directory_operations.py @@ -4,10 +4,10 @@ from unittest.mock import MagicMock import pytest -from freqtrade import OperationalException from freqtrade.configuration.directory_operations import (copy_sample_files, create_datadir, create_userdata_dir) +from freqtrade.exceptions import OperationalException from tests.conftest import log_has, log_has_re @@ -25,7 +25,7 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None: md = mocker.patch.object(Path, 'mkdir', MagicMock()) x = create_userdata_dir('/tmp/bar', create_dir=True) - assert md.call_count == 8 + assert md.call_count == 9 assert md.call_args[1]['parents'] is False assert log_has(f'Created user-data directory: {Path("/tmp/bar")}', caplog) assert isinstance(x, Path) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 341bc021f..9d9d133cc 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -11,9 +11,9 @@ import arrow import pytest import requests -from freqtrade import (DependencyException, InvalidOrderException, - OperationalException, TemporaryError, constants) -from freqtrade.constants import MATH_CLOSE_PREC +from freqtrade.constants import MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT, CANCEL_REASON +from freqtrade.exceptions import (DependencyException, InvalidOrderException, + OperationalException, TemporaryError) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Trade from freqtrade.rpc import RPCMessageType @@ -22,7 +22,7 @@ from freqtrade.strategy.interface import SellCheckTuple, SellType from freqtrade.worker import Worker from tests.conftest import (get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, - patch_get_signal, patch_wallet, patch_whitelist) + patch_get_signal, patch_wallet, patch_whitelist, create_mock_trades) def patch_RPCManager(mocker) -> MagicMock: @@ -48,13 +48,31 @@ def test_freqtradebot_state(mocker, default_conf, markets) -> None: assert freqtrade.state is State.STOPPED -def test_cleanup(mocker, default_conf, caplog) -> None: - mock_cleanup = MagicMock() - mocker.patch('freqtrade.persistence.cleanup', mock_cleanup) +def test_process_stopped(mocker, default_conf) -> None: + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + coo_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cancel_all_open_orders') + freqtrade.process_stopped() + assert coo_mock.call_count == 0 + + default_conf['cancel_open_orders_on_exit'] = True + freqtrade = get_patched_freqtradebot(mocker, default_conf) + freqtrade.process_stopped() + assert coo_mock.call_count == 1 + + +def test_bot_cleanup(mocker, default_conf, caplog) -> None: + mock_cleanup = mocker.patch('freqtrade.persistence.cleanup') + coo_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cancel_all_open_orders') freqtrade = get_patched_freqtradebot(mocker, default_conf) freqtrade.cleanup() assert log_has('Cleaning up modules ...', caplog) assert mock_cleanup.call_count == 1 + assert coo_mock.call_count == 0 + + freqtrade.config['cancel_open_orders_on_exit'] = True + freqtrade.cleanup() + assert coo_mock.call_count == 1 def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: @@ -136,60 +154,106 @@ def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: freqtrade = FreqtradeBot(default_conf) - result = freqtrade._get_trade_stake_amount('ETH/BTC') + result = freqtrade.get_trade_stake_amount('ETH/BTC') assert result == default_conf['stake_amount'] +@pytest.mark.parametrize("amend_last,wallet,max_open,lsamr,expected", [ + (False, 0.002, 2, 0.5, [0.001, None]), + (True, 0.002, 2, 0.5, [0.001, 0.00098]), + (False, 0.003, 3, 0.5, [0.001, 0.001, None]), + (True, 0.003, 3, 0.5, [0.001, 0.001, 0.00097]), + (False, 0.0022, 3, 0.5, [0.001, 0.001, None]), + (True, 0.0022, 3, 0.5, [0.001, 0.001, 0.0]), + (True, 0.0027, 3, 0.5, [0.001, 0.001, 0.000673]), + (True, 0.0022, 3, 1, [0.001, 0.001, 0.0]), + ]) +def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order, + amend_last, wallet, max_open, lsamr, expected) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_balance=MagicMock(return_value=default_conf['stake_amount'] * 2), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee + ) + default_conf['dry_run_wallet'] = wallet + + default_conf['amend_last_stake_amount'] = amend_last + default_conf['last_stake_amount_min_ratio'] = lsamr + + freqtrade = FreqtradeBot(default_conf) + + for i in range(0, max_open): + + if expected[i] is not None: + result = freqtrade.get_trade_stake_amount('ETH/BTC') + assert pytest.approx(result) == expected[i] + freqtrade.execute_buy('ETH/BTC', result) + else: + with pytest.raises(DependencyException): + freqtrade.get_trade_stake_amount('ETH/BTC') + + def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5) freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade._get_trade_stake_amount('ETH/BTC') + freqtrade.get_trade_stake_amount('ETH/BTC') -def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, +@pytest.mark.parametrize("balance_ratio,result1", [ + (1, 0.005), + (0.99, 0.00495), + (0.50, 0.0025), + ]) +def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_ratio, result1, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - patch_wallet(mocker, free=default_conf['stake_amount']) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee ) conf = deepcopy(default_conf) - conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + conf['stake_amount'] = UNLIMITED_STAKE_AMOUNT + conf['dry_run_wallet'] = 0.01 conf['max_open_trades'] = 2 + conf['tradable_balance_ratio'] = balance_ratio freqtrade = FreqtradeBot(conf) patch_get_signal(freqtrade) # no open trades, order amount should be 'balance / max_open_trades' - result = freqtrade._get_trade_stake_amount('ETH/BTC') - assert result == default_conf['stake_amount'] / conf['max_open_trades'] + result = freqtrade.get_trade_stake_amount('ETH/BTC') + assert result == result1 # create one trade, order amount should be 'balance / (max_open_trades - num_open_trades)' freqtrade.execute_buy('ETH/BTC', result) - result = freqtrade._get_trade_stake_amount('LTC/BTC') - assert result == default_conf['stake_amount'] / (conf['max_open_trades'] - 1) + result = freqtrade.get_trade_stake_amount('LTC/BTC') + assert result == result1 # create 2 trades, order amount should be None freqtrade.execute_buy('LTC/BTC', result) - result = freqtrade._get_trade_stake_amount('XRP/BTC') - assert result is None + result = freqtrade.get_trade_stake_amount('XRP/BTC') + assert result == 0 # set max_open_trades = None, so do not trade conf['max_open_trades'] = 0 freqtrade = FreqtradeBot(conf) - result = freqtrade._get_trade_stake_amount('NEO/BTC') - assert result is None + result = freqtrade.get_trade_stake_amount('NEO/BTC') + assert result == 0 def test_edge_called_in_process(mocker, edge_conf) -> None: @@ -214,8 +278,8 @@ def test_edge_overrides_stake_amount(mocker, edge_conf) -> None: edge_conf['dry_run_wallet'] = 999.9 freqtrade = FreqtradeBot(edge_conf) - assert freqtrade._get_trade_stake_amount('NEO/BTC') == (999.9 * 0.5 * 0.01) / 0.20 - assert freqtrade._get_trade_stake_amount('LTC/BTC') == (999.9 * 0.5 * 0.01) / 0.21 + assert freqtrade.get_trade_stake_amount('NEO/BTC') == (999.9 * 0.5 * 0.01) / 0.20 + assert freqtrade.get_trade_stake_amount('LTC/BTC') == (999.9 * 0.5 * 0.01) / 0.21 def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf) -> None: @@ -232,7 +296,7 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf buy_price = limit_buy_order['price'] mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price * 0.79, 'ask': buy_price * 0.79, 'last': buy_price * 0.79 @@ -247,14 +311,14 @@ def test_edge_overrides_stoploss(limit_buy_order, fee, caplog, mocker, edge_conf freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) ############################################# # stoploss shoud be hit assert freqtrade.handle_trade(trade) is True - assert log_has('executed sell, reason: SellType.STOP_LOSS', caplog) + assert log_has('Executing Sell for NEO/BTC. Reason: SellType.STOP_LOSS', caplog) assert trade.sell_reason == SellType.STOP_LOSS.value @@ -272,7 +336,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, buy_price = limit_buy_order['price'] mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price * 0.85, 'ask': buy_price * 0.85, 'last': buy_price * 0.85 @@ -287,7 +351,7 @@ def test_edge_should_ignore_strategy_stoploss(limit_buy_order, fee, freqtrade.active_pair_whitelist = ['NEO/BTC'] patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) ############################################# @@ -304,13 +368,13 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, default_conf['max_open_trades'] = 2 mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade is not None @@ -318,7 +382,7 @@ def test_total_open_trades_stakes(mocker, default_conf, ticker, assert trade.is_open assert trade.open_date is not None - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.order_by(Trade.id.desc()).first() assert trade is not None @@ -462,12 +526,12 @@ def test_get_min_pair_stake_amount_real_data(mocker, default_conf) -> None: assert round(result, 8) == round(max(0.0001, 0.001 * 0.020405) / 0.9, 8) -def test_create_trades(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_create_trade(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -476,7 +540,7 @@ def test_create_trades(default_conf, ticker, limit_buy_order, fee, mocker) -> No whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.create_trade('ETH/BTC') trade = Trade.query.first() assert trade is not None @@ -494,14 +558,14 @@ def test_create_trades(default_conf, ticker, limit_buy_order, fee, mocker) -> No assert whitelist == default_conf['exchange']['pair_whitelist'] -def test_create_trades_no_stake_amount(default_conf, ticker, limit_buy_order, - fee, mocker) -> None: +def test_create_trade_no_stake_amount(default_conf, ticker, limit_buy_order, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) patch_wallet(mocker, free=default_conf['stake_amount'] * 0.5) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -509,17 +573,17 @@ def test_create_trades_no_stake_amount(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade) with pytest.raises(DependencyException, match=r'.*stake amount.*'): - freqtrade.create_trades() + freqtrade.create_trade('ETH/BTC') -def test_create_trades_minimal_amount(default_conf, ticker, limit_buy_order, - fee, mocker) -> None: +def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=buy_mock, get_fee=fee, ) @@ -527,19 +591,19 @@ def test_create_trades_minimal_amount(default_conf, ticker, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.create_trade('ETH/BTC') rate, amount = buy_mock.call_args[1]['rate'], buy_mock.call_args[1]['amount'] assert rate * amount >= default_conf['stake_amount'] -def test_create_trades_too_small_stake_amount(default_conf, ticker, limit_buy_order, - fee, mocker) -> None: +def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order, + fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) buy_mock = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=buy_mock, get_fee=fee, ) @@ -549,37 +613,37 @@ def test_create_trades_too_small_stake_amount(default_conf, ticker, limit_buy_or patch_get_signal(freqtrade) - assert not freqtrade.create_trades() + assert not freqtrade.create_trade('ETH/BTC') -def test_create_trades_limit_reached(default_conf, ticker, limit_buy_order, - fee, markets, mocker) -> None: +def test_create_trade_limit_reached(default_conf, ticker, limit_buy_order, + fee, markets, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_balance=MagicMock(return_value=default_conf['stake_amount']), get_fee=fee, ) default_conf['max_open_trades'] = 0 - default_conf['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT + default_conf['stake_amount'] = UNLIMITED_STAKE_AMOUNT freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - assert not freqtrade.create_trades() - assert freqtrade._get_trade_stake_amount('ETH/BTC') is None + assert not freqtrade.create_trade('ETH/BTC') + assert freqtrade.get_trade_stake_amount('ETH/BTC') == 0 -def test_create_trades_no_pairs_let(default_conf, ticker, limit_buy_order, fee, - mocker, caplog) -> None: +def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order, fee, + mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -588,19 +652,21 @@ def test_create_trades_no_pairs_let(default_conf, ticker, limit_buy_order, fee, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - assert freqtrade.create_trades() - assert not freqtrade.create_trades() - assert log_has("No currency pair in active pair whitelist, " - "but checking to sell open trades.", caplog) + n = freqtrade.enter_positions() + assert n == 1 + assert not log_has_re(r"No currency pair in active pair whitelist.*", caplog) + n = freqtrade.enter_positions() + assert n == 0 + assert log_has_re(r"No currency pair in active pair whitelist.*", caplog) -def test_create_trades_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, - mocker, caplog) -> None: +def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, + mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -608,11 +674,12 @@ def test_create_trades_no_pairs_in_whitelist(default_conf, ticker, limit_buy_ord freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - assert not freqtrade.create_trades() + n = freqtrade.enter_positions() + assert n == 0 assert log_has("Active pair whitelist is empty.", caplog) -def test_create_trades_no_signal(default_conf, fee, mocker) -> None: +def test_create_trade_no_signal(default_conf, fee, mocker) -> None: default_conf['dry_run'] = True patch_RPCManager(mocker) @@ -628,28 +695,34 @@ def test_create_trades_no_signal(default_conf, fee, mocker) -> None: Trade.query = MagicMock() Trade.query.filter = MagicMock() - assert not freqtrade.create_trades() + assert not freqtrade.create_trade('ETH/BTC') @pytest.mark.parametrize("max_open", range(0, 5)) -def test_create_trades_multiple_trades(default_conf, ticker, - fee, mocker, max_open) -> None: +@pytest.mark.parametrize("tradable_balance_ratio,modifier", [(1.0, 1), (0.99, 0.8), (0.5, 0.5)]) +def test_create_trades_multiple_trades(default_conf, ticker, fee, mocker, + max_open, tradable_balance_ratio, modifier) -> None: patch_RPCManager(mocker) patch_exchange(mocker) default_conf['max_open_trades'] = max_open + default_conf['tradable_balance_ratio'] = tradable_balance_ratio + default_conf['dry_run_wallet'] = 0.001 * max_open + mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': "12355555"}), get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trades() - + n = freqtrade.enter_positions() trades = Trade.get_open_trades() - assert len(trades) == max_open + # Expected trades should be max_open * a modified value + # depending on the configured tradable_balance + assert n == max(int(max_open * modifier), 0) + assert len(trades) == max(int(max_open * modifier), 0) def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: @@ -658,7 +731,7 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: default_conf['max_open_trades'] = 4 mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': "12355555"}), get_fee=fee, ) @@ -672,7 +745,8 @@ def test_create_trades_preopen(default_conf, ticker, fee, mocker) -> None: assert len(Trade.get_open_trades()) == 2 # Create 2 new trades using create_trades - assert freqtrade.create_trades() + assert freqtrade.create_trade('ETH/BTC') + assert freqtrade.create_trade('NEO/BTC') trades = Trade.get_open_trades() assert len(trades) == 4 @@ -684,7 +758,7 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, @@ -705,8 +779,8 @@ def test_process_trade_creation(default_conf, ticker, limit_buy_order, assert trade.is_open assert trade.open_date is not None assert trade.exchange == 'bittrex' - assert trade.open_rate == 0.00001099 - assert trade.amount == 90.99181073703367 + assert trade.open_rate == 0.00001098 + assert trade.amount == 91.07468123861567 assert log_has( 'Buy signal found: about create a new trade with stake_amount: 0.001 ...', caplog @@ -718,7 +792,7 @@ def test_process_exchange_failures(default_conf, ticker, mocker) -> None: patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(side_effect=TemporaryError) ) sleep_mock = mocker.patch('time.sleep', side_effect=lambda _: None) @@ -726,7 +800,7 @@ def test_process_exchange_failures(default_conf, ticker, mocker) -> None: worker = Worker(args=None, config=default_conf) patch_get_signal(worker.freqtrade) - worker._process() + worker._process_running() assert sleep_mock.has_calls() @@ -735,16 +809,16 @@ def test_process_operational_exception(default_conf, ticker, mocker) -> None: patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(side_effect=OperationalException) ) worker = Worker(args=None, config=default_conf) patch_get_signal(worker.freqtrade) - assert worker.state == State.RUNNING + assert worker.freqtrade.state == State.RUNNING - worker._process() - assert worker.state == State.STOPPED + worker._process_running() + assert worker.freqtrade.state == State.STOPPED assert 'OperationalException' in msg_mock.call_args_list[-1][0][0]['status'] @@ -753,7 +827,7 @@ def test_process_trade_handling(default_conf, ticker, limit_buy_order, fee, mock patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, @@ -780,7 +854,7 @@ def test_process_trade_no_whitelist_pair(default_conf, ticker, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_order=MagicMock(return_value=limit_buy_order), get_fee=fee, @@ -830,7 +904,7 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: refresh_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(side_effect=TemporaryError), refresh_latest_ohlcv=refresh_mock, ) @@ -850,30 +924,47 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert ("ETH/BTC", default_conf["ticker_interval"]) in refresh_mock.call_args[0][0] -def test_balance_fully_ask_side(mocker, default_conf) -> None: - default_conf['bid_strategy']['ask_last_balance'] = 0.0 +@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ + ('ask', 20, 19, 10, 0.0, 20), # Full ask side + ('ask', 20, 19, 10, 1.0, 10), # Full last side + ('ask', 20, 19, 10, 0.5, 15), # Between ask and last + ('ask', 20, 19, 10, 0.7, 13), # Between ask and last + ('ask', 20, 19, 10, 0.3, 17), # Between ask and last + ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask + ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask + ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask + ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask + ('ask', 4, 5, None, 1, 4), # last not available - uses ask + ('ask', 4, 5, None, 0, 4), # last not available - uses ask + ('bid', 10, 20, 10, 0.0, 20), # Full bid side + ('bid', 10, 20, 10, 1.0, 10), # Full last side + ('bid', 10, 20, 10, 0.5, 15), # Between bid and last + ('bid', 10, 20, 10, 0.7, 13), # Between bid and last + ('bid', 10, 20, 10, 0.3, 17), # Between bid and last + ('bid', 4, 5, 10, 1.0, 5), # last bigger than bid + ('bid', 4, 5, 10, 0.5, 5), # last bigger than bid + ('bid', 10, 20, None, 0.5, 20), # last not available - uses bid + ('bid', 4, 5, None, 0.5, 5), # last not available - uses bid + ('bid', 4, 5, None, 1, 5), # last not available - uses bid + ('bid', 4, 5, None, 0, 5), # last not available - uses bid +]) +def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, + last, last_ab, expected) -> None: + default_conf['bid_strategy']['ask_last_balance'] = last_ab + default_conf['bid_strategy']['price_side'] = side freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', - MagicMock(return_value={'ask': 20, 'last': 10})) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + MagicMock(return_value={'ask': ask, 'last': last, 'bid': bid})) - assert freqtrade.get_target_bid('ETH/BTC') == 20 + assert freqtrade.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) - -def test_balance_fully_last_side(mocker, default_conf) -> None: - default_conf['bid_strategy']['ask_last_balance'] = 1.0 - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', - MagicMock(return_value={'ask': 20, 'last': 10})) - - assert freqtrade.get_target_bid('ETH/BTC') == 10 - - -def test_balance_bigger_last_ask(mocker, default_conf) -> None: - default_conf['bid_strategy']['ask_last_balance'] = 1.0 - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', - MagicMock(return_value={'ask': 5, 'last': 10})) - assert freqtrade.get_target_bid('ETH/BTC') == 5 + assert freqtrade.get_buy_rate('ETH/BTC', False) == expected + assert log_has("Using cached buy rate for ETH/BTC.", caplog) + # Running a 2nd time with Refresh on! + caplog.clear() + assert freqtrade.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: @@ -882,16 +973,16 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: freqtrade = FreqtradeBot(default_conf) stake_amount = 2 bid = 0.11 - get_bid = MagicMock(return_value=bid) + buy_rate_mock = MagicMock(return_value=bid) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', - get_target_bid=get_bid, + get_buy_rate=buy_rate_mock, _get_min_pair_stake_amount=MagicMock(return_value=1) - ) + ) buy_mm = MagicMock(return_value={'id': limit_buy_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -902,7 +993,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: pair = 'ETH/BTC' assert freqtrade.execute_buy(pair, stake_amount) - assert get_bid.call_count == 1 + assert buy_rate_mock.call_count == 1 assert buy_mm.call_count == 1 call_args = buy_mm.call_args_list[0][1] assert call_args['pair'] == pair @@ -919,8 +1010,8 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order) -> None: # Test calling with price fix_price = 0.06 assert freqtrade.execute_buy(pair, stake_amount, fix_price) - # Make sure get_target_bid wasn't called again - assert get_bid.call_count == 1 + # Make sure get_buy_rate wasn't called again + assert buy_rate_mock.call_count == 1 assert buy_mm.call_count == 2 call_args = buy_mm.call_args_list[1][1] @@ -975,8 +1066,8 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=limit_buy_order['amount']) - stoploss_limit = MagicMock(return_value={'id': 13434334}) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + stoploss = MagicMock(return_value={'id': 13434334}) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -987,20 +1078,20 @@ def test_add_stoploss_on_exchange(mocker, default_conf, limit_buy_order) -> None trade.is_open = True trades = [trade] - freqtrade.process_maybe_execute_sells(trades) + freqtrade.exit_positions(trades) assert trade.stoploss_order_id == '13434334' - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 assert trade.is_open is True def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, limit_buy_order, limit_sell_order) -> None: - stoploss_limit = MagicMock(return_value={'id': 13434334}) + stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1008,7 +1099,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss_limit=stoploss_limit + stoploss=stoploss ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) @@ -1022,7 +1113,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, trade.stoploss_order_id = None assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 assert trade.stoploss_order_id == "13434334" # Second case: when stoploss is set but it is not yet hit @@ -1046,17 +1137,17 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, canceled_stoploss_order = MagicMock(return_value={'status': 'canceled'}) mocker.patch('freqtrade.exchange.Exchange.get_order', canceled_stoploss_order) - stoploss_limit.reset_mock() + stoploss.reset_mock() assert freqtrade.handle_stoploss_on_exchange(trade) is False - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 assert trade.stoploss_order_id == "13434334" # Fourth case: when stoploss is set and it is hit # should unset stoploss_order_id and return true # as a trade actually happened caplog.clear() - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1067,7 +1158,8 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, 'status': 'closed', 'type': 'stop_loss_limit', 'price': 3, - 'average': 2 + 'average': 2, + 'amount': limit_buy_order['amount'], }) mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_order_hit) assert freqtrade.handle_stoploss_on_exchange(trade) is True @@ -1076,9 +1168,10 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, assert trade.is_open is False mocker.patch( - 'freqtrade.exchange.Exchange.stoploss_limit', + 'freqtrade.exchange.Exchange.stoploss', side_effect=DependencyException() ) + trade.is_open = True freqtrade.handle_stoploss_on_exchange(trade) assert log_has('Unable to place a stoploss order on exchange.', caplog) assert trade.stoploss_order_id is None @@ -1086,11 +1179,21 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, # Fifth case: get_order returns InvalidOrder # It should try to add stoploss order trade.stoploss_order_id = 100 - stoploss_limit.reset_mock() + stoploss.reset_mock() mocker.patch('freqtrade.exchange.Exchange.get_order', side_effect=InvalidOrderException()) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) freqtrade.handle_stoploss_on_exchange(trade) - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 + + # Sixth case: Closed Trade + # Should not create new order + trade.stoploss_order_id = None + trade.is_open = False + stoploss.reset_mock() + mocker.patch('freqtrade.exchange.Exchange.get_order') + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss) + assert freqtrade.handle_stoploss_on_exchange(trade) is False + assert stoploss.call_count == 0 def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, @@ -1100,7 +1203,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1109,15 +1212,15 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, get_order=MagicMock(return_value={'status': 'canceled'}), - stoploss_limit=MagicMock(side_effect=DependencyException()), + stoploss=MagicMock(side_effect=DependencyException()), ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.is_open = True - trade.open_order_id = '12345' + trade.open_order_id = None trade.stoploss_order_id = 100 assert trade @@ -1134,7 +1237,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, sell_mock = MagicMock(return_value={'id': limit_sell_order['id']}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1143,13 +1246,13 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, sell=sell_mock, get_fee=fee, get_order=MagicMock(return_value={'status': 'canceled'}), - stoploss_limit=MagicMock(side_effect=InvalidOrderException()), + stoploss=MagicMock(side_effect=InvalidOrderException()), ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.order_types['stoploss_on_exchange'] = True - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() caplog.clear() freqtrade.create_stoploss_order(trade, 200, 199) @@ -1173,11 +1276,11 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set - stoploss_limit = MagicMock(return_value={'id': 13434334}) + stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1185,7 +1288,8 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss_limit=stoploss_limit + stoploss=stoploss, + stoploss_adjust=MagicMock(return_value=True), ) # enabling TSL @@ -1207,7 +1311,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1231,7 +1335,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, assert freqtrade.handle_stoploss_on_exchange(trade) is False # price jumped 2x - mocker.patch('freqtrade.exchange.Exchange.get_ticker', MagicMock(return_value={ + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00002344, 'ask': 0.00002346, 'last': 0.00002344 @@ -1240,7 +1344,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock() mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order_mock) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_order_mock) + mocker.patch('freqtrade.exchange.Exchange.stoploss', stoploss_order_mock) # stoploss should not be updated as the interval is 60 seconds assert freqtrade.handle_trade(trade) is False @@ -1249,7 +1353,7 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, stoploss_order_mock.assert_not_called() assert freqtrade.handle_trade(trade) is False - assert trade.stop_loss == 0.00002344 * 0.95 + assert trade.stop_loss == 0.00002346 * 0.95 # setting stoploss_on_exchange_interval to 0 seconds freqtrade.strategy.order_types['stoploss_on_exchange_interval'] = 0 @@ -1257,21 +1361,29 @@ def test_handle_stoploss_on_exchange_trailing(mocker, default_conf, fee, caplog, assert freqtrade.handle_stoploss_on_exchange(trade) is False cancel_order_mock.assert_called_once_with(100, 'ETH/BTC') - stoploss_order_mock.assert_called_once_with(amount=85.25149190110828, + stoploss_order_mock.assert_called_once_with(amount=85.32423208191126, pair='ETH/BTC', - rate=0.00002344 * 0.95 * 0.99, - stop_price=0.00002344 * 0.95) + order_types=freqtrade.strategy.order_types, + stop_price=0.00002346 * 0.95) + + # price fell below stoploss, so dry-run sells trade. + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ + 'bid': 0.00002144, + 'ask': 0.00002146, + 'last': 0.00002144 + })) + assert freqtrade.handle_trade(trade) is True def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, caplog, limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set - stoploss_limit = MagicMock(return_value={'id': 13434334}) + stoploss = MagicMock(return_value={'id': 13434334}) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1279,7 +1391,8 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss_limit=stoploss_limit + stoploss=stoploss, + stoploss_adjust=MagicMock(return_value=True), ) # enabling TSL @@ -1295,7 +1408,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c # setting stoploss_on_exchange_interval to 60 seconds freqtrade.strategy.order_types['stoploss_on_exchange_interval'] = 60 patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1319,12 +1432,12 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c assert log_has_re(r"Could not cancel stoploss order abcd for pair ETH/BTC.*", caplog) # Still try to create order - assert stoploss_limit.call_count == 1 + assert stoploss.call_count == 1 # Fail creating stoploss order caplog.clear() cancel_mock = mocker.patch("freqtrade.exchange.Exchange.cancel_order", MagicMock()) - mocker.patch("freqtrade.exchange.Exchange.stoploss_limit", side_effect=DependencyException()) + mocker.patch("freqtrade.exchange.Exchange.stoploss", side_effect=DependencyException()) freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert cancel_mock.call_count == 1 assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) @@ -1334,15 +1447,16 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, limit_buy_order, limit_sell_order) -> None: # When trailing stoploss is set - stoploss_limit = MagicMock(return_value={'id': 13434334}) + stoploss = MagicMock(return_value={'id': 13434334}) patch_RPCManager(mocker) patch_exchange(mocker) patch_edge(mocker) edge_conf['max_open_trades'] = float('inf') edge_conf['dry_run_wallet'] = 999.9 + edge_conf['exchange']['name'] = 'binance' mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1350,7 +1464,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, buy=MagicMock(return_value={'id': limit_buy_order['id']}), sell=MagicMock(return_value={'id': limit_sell_order['id']}), get_fee=fee, - stoploss_limit=stoploss_limit + stoploss=stoploss, ) # enabling TSL @@ -1376,7 +1490,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, freqtrade.active_pair_whitelist = freqtrade.edge.adjust(freqtrade.active_pair_whitelist) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.is_open = True trade.open_order_id = None @@ -1403,10 +1517,10 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, cancel_order_mock = MagicMock() stoploss_order_mock = MagicMock() mocker.patch('freqtrade.exchange.Exchange.cancel_order', cancel_order_mock) - mocker.patch('freqtrade.exchange.Exchange.stoploss_limit', stoploss_order_mock) + mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss_order_mock) # price goes down 5% - mocker.patch('freqtrade.exchange.Exchange.get_ticker', MagicMock(return_value={ + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00001172 * 0.95, 'ask': 0.00001173 * 0.95, 'last': 0.00001172 * 0.95 @@ -1422,7 +1536,7 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, cancel_order_mock.assert_not_called() # price jumped 2x - mocker.patch('freqtrade.exchange.Exchange.get_ticker', MagicMock(return_value={ + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00002344, 'ask': 0.00002346, 'last': 0.00002344 @@ -1432,35 +1546,41 @@ def test_tsl_on_exchange_compatible_with_edge(mocker, edge_conf, fee, caplog, assert freqtrade.handle_stoploss_on_exchange(trade) is False # stoploss should be set to 1% as trailing is on - assert trade.stop_loss == 0.00002344 * 0.99 + assert trade.stop_loss == 0.00002346 * 0.99 cancel_order_mock.assert_called_once_with(100, 'NEO/BTC') - stoploss_order_mock.assert_called_once_with(amount=2131074.168797954, + stoploss_order_mock.assert_called_once_with(amount=2132892.491467577, pair='NEO/BTC', - rate=0.00002344 * 0.99 * 0.99, - stop_price=0.00002344 * 0.99) + order_types=freqtrade.strategy.order_types, + stop_price=0.00002346 * 0.99) -def test_process_maybe_execute_buys(mocker, default_conf, caplog) -> None: +def test_enter_positions(mocker, default_conf, caplog) -> None: caplog.set_level(logging.DEBUG) freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trades', MagicMock(return_value=False)) - freqtrade.process_maybe_execute_buys() + mock_ct = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.create_trade', + MagicMock(return_value=False)) + n = freqtrade.enter_positions() + assert n == 0 assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog) + # create_trade should be called once for every pair in the whitelist. + assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) -def test_process_maybe_execute_buys_exception(mocker, default_conf, caplog) -> None: +def test_enter_positions_exception(mocker, default_conf, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch( - 'freqtrade.freqtradebot.FreqtradeBot.create_trades', + mock_ct = mocker.patch( + 'freqtrade.freqtradebot.FreqtradeBot.create_trade', MagicMock(side_effect=DependencyException) ) - freqtrade.process_maybe_execute_buys() - assert log_has('Unable to create trade: ', caplog) + n = freqtrade.enter_positions() + assert n == 0 + assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) + assert log_has('Unable to create trade for ETH/BTC: ', caplog) -def test_process_maybe_execute_sells(mocker, default_conf, limit_buy_order, caplog) -> None: +def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_trade', MagicMock(return_value=True)) @@ -1473,7 +1593,8 @@ def test_process_maybe_execute_sells(mocker, default_conf, limit_buy_order, capl trade.open_order_id = '123' trade.open_fee = 0.001 trades = [trade] - assert not freqtrade.process_maybe_execute_sells(trades) + n = freqtrade.exit_positions(trades) + assert n == 0 # Test amount not modified by fee-logic assert not log_has( 'Applying fee to amount for Trade {} from 90.99181073 to 90.81'.format(trade), caplog @@ -1481,25 +1602,26 @@ def test_process_maybe_execute_sells(mocker, default_conf, limit_buy_order, capl mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=90.81) # test amount modified by fee-logic - assert not freqtrade.process_maybe_execute_sells(trades) + n = freqtrade.exit_positions(trades) + assert n == 0 -def test_process_maybe_execute_sells_exception(mocker, default_conf, - limit_buy_order, caplog) -> None: +def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.get_order', return_value=limit_buy_order) trade = MagicMock() - trade.open_order_id = '123' + trade.open_order_id = None trade.open_fee = 0.001 trades = [trade] # Test raise of DependencyException exception mocker.patch( - 'freqtrade.freqtradebot.FreqtradeBot.update_trade_state', + 'freqtrade.freqtradebot.FreqtradeBot.handle_trade', side_effect=DependencyException() ) - freqtrade.process_maybe_execute_sells(trades) + n = freqtrade.exit_positions(trades) + assert n == 0 assert log_has('Unable to sell trade: ', caplog) @@ -1662,7 +1784,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -1674,7 +1796,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade @@ -1682,6 +1804,7 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock time.sleep(0.01) # Race condition fix trade.update(limit_buy_order) assert trade.is_open is True + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True @@ -1696,12 +1819,12 @@ def test_handle_trade(default_conf, limit_buy_order, limit_sell_order, fee, mock assert trade.close_date is not None -def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, mocker) -> None: +def test_handle_overlapping_signals(default_conf, ticker, limit_buy_order, fee, mocker) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -1710,7 +1833,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, patch_get_signal(freqtrade, value=(True, True)) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() # Buy and Sell triggering, so doing nothing ... trades = Trade.query.all() @@ -1719,7 +1842,7 @@ def test_handle_overlpapping_signals(default_conf, ticker, limit_buy_order, fee, # Buy is triggering, so buying ... patch_get_signal(freqtrade, value=(True, False)) - freqtrade.create_trades() + freqtrade.enter_positions() trades = Trade.query.all() nb_trades = len(trades) assert nb_trades == 1 @@ -1754,7 +1877,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -1763,7 +1886,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order, patch_get_signal(freqtrade, value=(True, False)) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.is_open = True @@ -1786,7 +1909,7 @@ def test_handle_trade_use_sell_signal( patch_RPCManager(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -1794,7 +1917,7 @@ def test_handle_trade_use_sell_signal( freqtrade = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.is_open = True @@ -1814,7 +1937,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -1822,7 +1945,7 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, patch_get_signal(freqtrade) # Create trade and sell it - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade @@ -1835,14 +1958,16 @@ def test_close_trade(default_conf, ticker, limit_buy_order, limit_sell_order, freqtrade.handle_trade(trade) -def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, open_trade, - fee, mocker) -> None: +def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_order_old, open_trade, + fee, mocker) -> None: + default_conf["unfilledtimeout"] = {"buy": 1400, "sell": 30} + rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock(return_value=limit_buy_order_old) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old), cancel_order=cancel_order_mock, get_fee=fee @@ -1851,6 +1976,56 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, op Trade.session.add(open_trade) + # Ensure default is to return empty (so not mocked yet) + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 0 + + # Return false - trade remains open + freqtrade.strategy.check_buy_timeout = MagicMock(return_value=False) + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 0 + trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 1 + assert freqtrade.strategy.check_buy_timeout.call_count == 1 + + # Raise Keyerror ... (no impact on trade) + freqtrade.strategy.check_buy_timeout = MagicMock(side_effect=KeyError) + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 0 + trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 1 + assert freqtrade.strategy.check_buy_timeout.call_count == 1 + + freqtrade.strategy.check_buy_timeout = MagicMock(return_value=True) + # Trade should be closed since the function returns true + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 1 + assert rpc_mock.call_count == 1 + trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() + nb_trades = len(trades) + assert nb_trades == 0 + assert freqtrade.strategy.check_buy_timeout.call_count == 1 + + +def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, open_trade, + fee, mocker) -> None: + rpc_mock = patch_RPCManager(mocker) + cancel_order_mock = MagicMock(return_value=limit_buy_order_old) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_order=MagicMock(return_value=limit_buy_order_old), + cancel_order_with_result=cancel_order_mock, + get_fee=fee + ) + freqtrade = FreqtradeBot(default_conf) + + Trade.session.add(open_trade) + + freqtrade.strategy.check_buy_timeout = MagicMock(return_value=False) # check it does cancel buy orders over the time limit freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 @@ -1858,6 +2033,8 @@ def test_check_handle_timedout_buy(default_conf, ticker, limit_buy_order_old, op trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() nb_trades = len(trades) assert nb_trades == 0 + # Custom user buy-timeout is never called + assert freqtrade.strategy.check_buy_timeout.call_count == 0 def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, open_trade, @@ -1866,10 +2043,10 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() patch_exchange(mocker) - limit_buy_order_old.update({"status": "canceled"}) + limit_buy_order_old.update({"status": "canceled", 'filled': 0.0}) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old), cancel_order=cancel_order_mock, get_fee=fee @@ -1885,7 +2062,7 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() nb_trades = len(trades) assert nb_trades == 0 - assert log_has_re("Buy order canceled on Exchange for Trade.*", caplog) + assert log_has_re("Buy order cancelled on exchange for Trade.*", caplog) def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, open_trade, @@ -1896,7 +2073,7 @@ def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_ord mocker.patch.multiple( 'freqtrade.exchange.Exchange', validate_pairs=MagicMock(), - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(side_effect=DependencyException), cancel_order=cancel_order_mock, get_fee=fee @@ -1914,6 +2091,54 @@ def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_ord assert nb_trades == 1 +def test_check_handle_timedout_sell_usercustom(default_conf, ticker, limit_sell_order_old, mocker, + open_trade) -> None: + default_conf["unfilledtimeout"] = {"buy": 1440, "sell": 1440} + rpc_mock = patch_RPCManager(mocker) + cancel_order_mock = MagicMock() + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_order=MagicMock(return_value=limit_sell_order_old), + cancel_order=cancel_order_mock + ) + freqtrade = FreqtradeBot(default_conf) + + open_trade.open_date = arrow.utcnow().shift(hours=-5).datetime + open_trade.close_date = arrow.utcnow().shift(minutes=-601).datetime + open_trade.is_open = False + + Trade.session.add(open_trade) + # Ensure default is false + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 0 + + freqtrade.strategy.check_sell_timeout = MagicMock(return_value=False) + # Return false - No impact + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 0 + assert rpc_mock.call_count == 0 + assert open_trade.is_open is False + assert freqtrade.strategy.check_sell_timeout.call_count == 1 + + freqtrade.strategy.check_sell_timeout = MagicMock(side_effect=KeyError) + # Return Error - No impact + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 0 + assert rpc_mock.call_count == 0 + assert open_trade.is_open is False + assert freqtrade.strategy.check_sell_timeout.call_count == 1 + + # Return True - sells! + freqtrade.strategy.check_sell_timeout = MagicMock(return_value=True) + freqtrade.check_handle_timedout() + assert cancel_order_mock.call_count == 1 + assert rpc_mock.call_count == 1 + assert open_trade.is_open is True + assert freqtrade.strategy.check_sell_timeout.call_count == 1 + + def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, mocker, open_trade) -> None: rpc_mock = patch_RPCManager(mocker) @@ -1921,7 +2146,7 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_sell_order_old), cancel_order=cancel_order_mock ) @@ -1933,11 +2158,14 @@ def test_check_handle_timedout_sell(default_conf, ticker, limit_sell_order_old, Trade.session.add(open_trade) + freqtrade.strategy.check_sell_timeout = MagicMock(return_value=False) # check it does cancel sell orders over the time limit freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 assert rpc_mock.call_count == 1 assert open_trade.is_open is True + # Custom user sell-timeout is never called + assert freqtrade.strategy.check_sell_timeout.call_count == 0 def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, open_trade, @@ -1945,13 +2173,13 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, """ Handle sell order cancelled on exchange""" rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock() - limit_sell_order_old.update({"status": "canceled"}) + limit_sell_order_old.update({"status": "canceled", 'filled': 0.0}) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_sell_order_old), - cancel_order=cancel_order_mock + cancel_order_with_result=cancel_order_mock ) freqtrade = FreqtradeBot(default_conf) @@ -1966,7 +2194,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, assert cancel_order_mock.call_count == 0 assert rpc_mock.call_count == 1 assert open_trade.is_open is True - assert log_has_re("Sell order canceled on exchange for Trade.*", caplog) + assert log_has_re("Sell order cancelled on exchange for Trade.*", caplog) def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, @@ -1976,9 +2204,9 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old_partial), - cancel_order=cancel_order_mock + cancel_order_with_result=cancel_order_mock ) freqtrade = FreqtradeBot(default_conf) @@ -1988,7 +2216,7 @@ def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old # note this is for a partially-complete buy order freqtrade.check_handle_timedout() assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 + assert rpc_mock.call_count == 2 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() assert len(trades) == 1 assert trades[0].amount == 23.0 @@ -2000,12 +2228,13 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap limit_buy_order_old_partial_canceled, mocker) -> None: rpc_mock = patch_RPCManager(mocker) cancel_order_mock = MagicMock(return_value=limit_buy_order_old_partial_canceled) + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=0)) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old_partial), - cancel_order=cancel_order_mock, + cancel_order_with_result=cancel_order_mock, get_trades_for_order=MagicMock(return_value=trades_for_order), ) freqtrade = FreqtradeBot(default_conf) @@ -2019,17 +2248,18 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap # and apply fees if necessary. freqtrade.check_handle_timedout() - assert log_has_re(r"Applying fee on amount for Trade.* Order", caplog) + assert log_has_re(r"Applying fee on amount for Trade.*", caplog) assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 + assert rpc_mock.call_count == 2 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() assert len(trades) == 1 - # Verify that tradehas been updated + # Verify that trade has been updated assert trades[0].amount == (limit_buy_order_old_partial['amount'] - - limit_buy_order_old_partial['remaining']) - 0.0001 + limit_buy_order_old_partial['remaining']) - 0.023 assert trades[0].open_order_id is None - assert trades[0].fee_open == 0 + assert trades[0].fee_updated('buy') + assert pytest.approx(trades[0].fee_open) == 0.001 def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee, @@ -2040,9 +2270,9 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(return_value=limit_buy_order_old_partial), - cancel_order=cancel_order_mock, + cancel_order_with_result=cancel_order_mock, get_trades_for_order=MagicMock(return_value=trades_for_order), ) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', @@ -2061,10 +2291,10 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, assert log_has_re(r"Could not update trade amount: .*", caplog) assert cancel_order_mock.call_count == 1 - assert rpc_mock.call_count == 1 + assert rpc_mock.call_count == 2 trades = Trade.query.filter(Trade.open_order_id.is_(open_trade.open_order_id)).all() assert len(trades) == 1 - # Verify that tradehas been updated + # Verify that trade has been updated assert trades[0].amount == (limit_buy_order_old_partial['amount'] - limit_buy_order_old_partial['remaining']) @@ -2079,12 +2309,12 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', - handle_timedout_limit_buy=MagicMock(), - handle_timedout_limit_sell=MagicMock(), + handle_cancel_buy=MagicMock(), + handle_cancel_sell=MagicMock(), ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_order=MagicMock(side_effect=requests.exceptions.RequestException('Oh snap')), cancel_order=cancel_order_mock ) @@ -2100,73 +2330,147 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke caplog) -def test_handle_timedout_limit_buy(mocker, default_conf, limit_buy_order) -> None: +def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock(return_value=limit_buy_order) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - cancel_order=cancel_order_mock - ) + mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) freqtrade = FreqtradeBot(default_conf) + freqtrade._notify_buy_cancel = MagicMock() Trade.session = MagicMock() trade = MagicMock() - limit_buy_order['remaining'] = limit_buy_order['amount'] - assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order) + trade.pair = 'LTC/ETH' + limit_buy_order['filled'] = 0.0 + limit_buy_order['status'] = 'open' + reason = CANCEL_REASON['TIMEOUT'] + assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() - limit_buy_order['amount'] = 2 - assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order) + limit_buy_order['filled'] = 2 + assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 + limit_buy_order['filled'] = 2 + mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException) + assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) -def test_handle_timedout_limit_buy_corder_empty(mocker, default_conf, limit_buy_order) -> None: + +@pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], + indirect=['limit_buy_order_canceled_empty']) +def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, + limit_buy_order_canceled_empty) -> None: patch_RPCManager(mocker) patch_exchange(mocker) - cancel_order_mock = MagicMock(return_value={}) + cancel_order_mock = mocker.patch( + 'freqtrade.exchange.Exchange.cancel_order_with_result', + return_value=limit_buy_order_canceled_empty) + nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_buy_cancel') + freqtrade = FreqtradeBot(default_conf) + + Trade.session = MagicMock() + reason = CANCEL_REASON['TIMEOUT'] + trade = MagicMock() + trade.pair = 'LTC/ETH' + assert freqtrade.handle_cancel_buy(trade, limit_buy_order_canceled_empty, reason) + assert cancel_order_mock.call_count == 0 + assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog) + assert nofiy_mock.call_count == 1 + + +@pytest.mark.parametrize('cancelorder', [ + {}, + {'remaining': None}, + 'String Return value', + 123 +]) +def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order, + cancelorder) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + cancel_order_mock = MagicMock(return_value=cancelorder) mocker.patch.multiple( 'freqtrade.exchange.Exchange', cancel_order=cancel_order_mock ) freqtrade = FreqtradeBot(default_conf) + freqtrade._notify_buy_cancel = MagicMock() Trade.session = MagicMock() trade = MagicMock() - limit_buy_order['remaining'] = limit_buy_order['amount'] - assert freqtrade.handle_timedout_limit_buy(trade, limit_buy_order) + trade.pair = 'LTC/ETH' + limit_buy_order['filled'] = 0.0 + limit_buy_order['status'] = 'open' + reason = CANCEL_REASON['TIMEOUT'] + assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() - limit_buy_order['amount'] = 2 - assert not freqtrade.handle_timedout_limit_buy(trade, limit_buy_order) + limit_buy_order['filled'] = 1.0 + assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 -def test_handle_timedout_limit_sell(mocker, default_conf) -> None: - patch_RPCManager(mocker) +def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: + send_msg_mock = patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() mocker.patch.multiple( 'freqtrade.exchange.Exchange', - cancel_order=cancel_order_mock + cancel_order=cancel_order_mock, ) + mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', return_value=0.245441) + + freqtrade = FreqtradeBot(default_conf) + + trade = Trade( + pair='LTC/ETH', + amount=2, + exchange='binance', + open_rate=0.245441, + open_order_id="123456", + open_date=arrow.utcnow().datetime, + fee_open=fee.return_value, + fee_close=fee.return_value, + ) + order = {'remaining': 1, + 'amount': 1, + 'status': "open"} + reason = CANCEL_REASON['TIMEOUT'] + assert freqtrade.handle_cancel_sell(trade, order, reason) + assert cancel_order_mock.call_count == 1 + assert send_msg_mock.call_count == 1 + + send_msg_mock.reset_mock() + + order['amount'] = 2 + assert freqtrade.handle_cancel_sell(trade, order, reason) == CANCEL_REASON['PARTIALLY_FILLED'] + # Assert cancel_order was not called (callcount remains unchanged) + assert cancel_order_mock.call_count == 1 + assert send_msg_mock.call_count == 1 + assert freqtrade.handle_cancel_sell(trade, order, reason) == CANCEL_REASON['PARTIALLY_FILLED'] + # Message should not be iterated again + assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED'] + assert send_msg_mock.call_count == 1 + + +def test_handle_cancel_sell_cancel_exception(mocker, default_conf) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch( + 'freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException()) freqtrade = FreqtradeBot(default_conf) trade = MagicMock() + reason = CANCEL_REASON['TIMEOUT'] order = {'remaining': 1, 'amount': 1, 'status': "open"} - assert freqtrade.handle_timedout_limit_sell(trade, order) - assert cancel_order_mock.call_count == 1 - order['amount'] = 2 - assert not freqtrade.handle_timedout_limit_sell(trade, order) - # Assert cancel_order was not called (callcount remains unchanged) - assert cancel_order_mock.call_count == 1 + assert freqtrade.handle_cancel_sell(trade, order, reason) == 'error cancelling order' def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None: @@ -2174,7 +2478,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -2182,7 +2486,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade @@ -2190,7 +2494,7 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI) @@ -2203,12 +2507,12 @@ def test_execute_sell_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> N 'pair': 'ETH/BTC', 'gain': 'profit', 'limit': 1.172e-05, - 'amount': 90.99181073703367, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.172e-05, - 'profit_amount': 6.126e-05, - 'profit_percent': 0.0611052, + 'open_rate': 1.098e-05, + 'current_rate': 1.173e-05, + 'profit_amount': 6.223e-05, + 'profit_ratio': 0.0620716, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.ROI.value, @@ -2222,7 +2526,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -2230,7 +2534,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade @@ -2238,7 +2542,7 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_down + fetch_ticker=ticker_sell_down ) freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], @@ -2252,12 +2556,12 @@ def test_execute_sell_down(default_conf, ticker, fee, ticker_sell_down, mocker) 'pair': 'ETH/BTC', 'gain': 'loss', 'limit': 1.044e-05, - 'amount': 90.99181073703367, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.044e-05, - 'profit_amount': -5.492e-05, - 'profit_percent': -0.05478342, + 'open_rate': 1.098e-05, + 'current_rate': 1.043e-05, + 'profit_amount': -5.406e-05, + 'profit_ratio': -0.05392257, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.STOP_LOSS.value, @@ -2272,7 +2576,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -2280,7 +2584,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade @@ -2288,7 +2592,7 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_down + fetch_ticker=ticker_sell_down ) default_conf['dry_run'] = True @@ -2308,12 +2612,12 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe 'pair': 'ETH/BTC', 'gain': 'loss', 'limit': 1.08801e-05, - 'amount': 90.99181073703367, + 'amount': 91.07468123861567, 'order_type': 'limit', - 'open_rate': 1.099e-05, - 'current_rate': 1.044e-05, - 'profit_amount': -1.498e-05, - 'profit_percent': -0.01493766, + 'open_rate': 1.098e-05, + 'current_rate': 1.043e-05, + 'profit_amount': -1.408e-05, + 'profit_ratio': -0.01404051, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.STOP_LOSS.value, @@ -2326,18 +2630,19 @@ def test_execute_sell_down_stoploss_on_exchange_dry_run(default_conf, ticker, fe def test_execute_sell_sloe_cancel_exception(mocker, default_conf, ticker, fee, caplog) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.cancel_order', side_effect=InvalidOrderException()) + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=300)) sellmock = MagicMock() patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, sell=sellmock ) freqtrade.strategy.order_types['stoploss_on_exchange'] = True patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() Trade.session = MagicMock() @@ -2357,7 +2662,7 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke default_conf['exchange']['name'] = 'binance' rpc_mock = patch_RPCManager(mocker) patch_exchange(mocker) - stoploss_limit = MagicMock(return_value={ + stoploss = MagicMock(return_value={ 'id': 123, 'info': { 'foo': 'bar' @@ -2367,11 +2672,11 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke cancel_order = MagicMock(return_value=True) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, - symbol_amount_prec=lambda s, x, y: y, - symbol_price_prec=lambda s, x, y: y, - stoploss_limit=stoploss_limit, + amount_to_precision=lambda s, x, y: y, + price_to_precision=lambda s, x, y: y, + stoploss=stoploss, cancel_order=cancel_order, ) @@ -2380,18 +2685,19 @@ def test_execute_sell_with_stoploss_on_exchange(default_conf, ticker, fee, ticke patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade trades = [trade] - freqtrade.process_maybe_execute_sells(trades) + freqtrade.check_handle_timedout() + freqtrade.exit_positions(trades) # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], @@ -2410,30 +2716,33 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, - symbol_amount_prec=lambda s, x, y: y, - symbol_price_prec=lambda s, x, y: y, + amount_to_precision=lambda s, x, y: y, + price_to_precision=lambda s, x, y: y, ) - stoploss_limit = MagicMock(return_value={ + stoploss = MagicMock(return_value={ 'id': 123, 'info': { 'foo': 'bar' } }) - mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) freqtrade = FreqtradeBot(default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trades() + freqtrade.enter_positions() + freqtrade.check_handle_timedout() trade = Trade.query.first() trades = [trade] - freqtrade.process_maybe_execute_sells(trades) + assert trade.stoploss_order_id is None + + freqtrade.exit_positions(trades) assert trade assert trade.stoploss_order_id == '123' assert trade.open_order_id is None @@ -2441,7 +2750,7 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f # Assuming stoploss on exchnage is hit # stoploss_order_id should become None # and trade should be sold at the price of stoploss - stoploss_limit_executed = MagicMock(return_value={ + stoploss_executed = MagicMock(return_value={ "id": "123", "timestamp": 1542707426845, "datetime": "2018-11-20T09:50:26.845Z", @@ -2459,9 +2768,9 @@ def test_may_execute_sell_after_stoploss_on_exchange_hit(default_conf, ticker, f "fee": None, "trades": None }) - mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_limit_executed) + mocker.patch('freqtrade.exchange.Exchange.get_order', stoploss_executed) - freqtrade.process_maybe_execute_sells(trades) + freqtrade.exit_positions(trades) assert trade.stoploss_order_id is None assert trade.is_open is False assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value @@ -2474,7 +2783,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) patch_whitelist(mocker, default_conf) @@ -2482,7 +2791,7 @@ def test_execute_sell_market_order(default_conf, ticker, fee, patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade @@ -2490,14 +2799,14 @@ def test_execute_sell_market_order(default_conf, ticker, fee, # Increase the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_up + fetch_ticker=ticker_sell_up ) freqtrade.config['order_types']['sell'] = 'market' freqtrade.execute_sell(trade=trade, limit=ticker_sell_up()['bid'], sell_reason=SellType.ROI) assert not trade.is_open - assert trade.close_profit == 0.0611052 + assert trade.close_profit == 0.0620716 assert rpc_mock.call_count == 2 last_msg = rpc_mock.call_args_list[-1][0][0] @@ -2507,12 +2816,12 @@ def test_execute_sell_market_order(default_conf, ticker, fee, 'pair': 'ETH/BTC', 'gain': 'profit', 'limit': 1.172e-05, - 'amount': 90.99181073703367, + 'amount': 91.07468123861567, 'order_type': 'market', - 'open_rate': 1.099e-05, - 'current_rate': 1.172e-05, - 'profit_amount': 6.126e-05, - 'profit_percent': 0.0611052, + 'open_rate': 1.098e-05, + 'current_rate': 1.173e-05, + 'profit_amount': 6.223e-05, + 'profit_ratio': 0.0620716, 'stake_currency': 'BTC', 'fiat_currency': 'USD', 'sell_reason': SellType.ROI.value, @@ -2528,7 +2837,7 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00002172, 'ask': 0.00002173, 'last': 0.00002172 @@ -2544,10 +2853,11 @@ def test_sell_profit_only_enable_profit(default_conf, limit_buy_order, patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True assert trade.sell_reason == SellType.SELL_SIGNAL.value @@ -2559,7 +2869,7 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00002172, 'ask': 0.00002173, 'last': 0.00002172 @@ -2574,10 +2884,11 @@ def test_sell_profit_only_disable_profit(default_conf, limit_buy_order, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True assert trade.sell_reason == SellType.SELL_SIGNAL.value @@ -2588,7 +2899,7 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00000172, 'ask': 0.00000173, 'last': 0.00000172 @@ -2603,8 +2914,8 @@ def test_sell_profit_only_enable_loss(default_conf, limit_buy_order, fee, mocker freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.stop_loss_reached = MagicMock(return_value=SellCheckTuple( - sell_flag=False, sell_type=SellType.NONE)) - freqtrade.create_trades() + sell_flag=False, sell_type=SellType.NONE)) + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2617,7 +2928,7 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.0000172, 'ask': 0.0000173, 'last': 0.0000172 @@ -2634,28 +2945,112 @@ def test_sell_profit_only_disable_loss(default_conf, limit_buy_order, fee, mocke patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True assert trade.sell_reason == SellType.SELL_SIGNAL.value +def test_sell_not_enough_balance(default_conf, limit_buy_order, + fee, mocker, caplog) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=MagicMock(return_value={ + 'bid': 0.00002172, + 'ask': 0.00002173, + 'last': 0.00002172 + }), + buy=MagicMock(return_value={'id': limit_buy_order['id']}), + get_fee=fee, + ) + + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) + + freqtrade.enter_positions() + + trade = Trade.query.first() + amnt = trade.amount + trade.update(limit_buy_order) + patch_get_signal(freqtrade, value=(False, True)) + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=trade.amount * 0.985)) + + assert freqtrade.handle_trade(trade) is True + assert log_has_re(r'.*Falling back to wallet-amount.', caplog) + assert trade.amount != amnt + + +def test__safe_sell_amount(default_conf, fee, caplog, mocker): + patch_RPCManager(mocker) + patch_exchange(mocker) + amount = 95.33 + amount_wallet = 95.29 + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=amount_wallet)) + wallet_update = mocker.patch('freqtrade.wallets.Wallets.update') + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456", + fee_open=fee.return_value, + fee_close=fee.return_value, + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + + wallet_update.reset_mock() + assert freqtrade._safe_sell_amount(trade.pair, trade.amount) == amount_wallet + assert log_has_re(r'.*Falling back to wallet-amount.', caplog) + assert wallet_update.call_count == 1 + caplog.clear() + wallet_update.reset_mock() + assert freqtrade._safe_sell_amount(trade.pair, amount_wallet) == amount_wallet + assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) + assert wallet_update.call_count == 1 + + +def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): + patch_RPCManager(mocker) + patch_exchange(mocker) + amount = 95.33 + amount_wallet = 91.29 + mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock(return_value=amount_wallet)) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + open_order_id="123456", + fee_open=fee.return_value, + fee_close=fee.return_value, + ) + freqtrade = FreqtradeBot(default_conf) + patch_get_signal(freqtrade) + with pytest.raises(DependencyException, match=r"Not enough amount to sell."): + assert freqtrade._safe_sell_amount(trade.pair, trade.amount) + + def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade @@ -2663,7 +3058,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo # Decrease the price and sell it mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker_sell_down + fetch_ticker=ticker_sell_down ) freqtrade.execute_sell(trade=trade, limit=ticker_sell_down()['bid'], @@ -2674,7 +3069,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo # reinit - should buy other pair. caplog.clear() - freqtrade.create_trades() + freqtrade.enter_positions() assert log_has(f"Pair {trade.pair} is currently locked.", caplog) @@ -2684,7 +3079,7 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.0000172, 'ask': 0.0000173, 'last': 0.0000172 @@ -2699,10 +3094,11 @@ def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, fee, mocker) -> patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) + freqtrade.wallets.update() patch_get_signal(freqtrade, value=(True, True)) assert freqtrade.handle_trade(trade) is False @@ -2717,7 +3113,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001099, 'ask': 0.00001099, 'last': 0.00001099 @@ -2731,12 +3127,12 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert freqtrade.handle_trade(trade) is False # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00001099 * 1.5, 'ask': 0.00001099 * 1.5, @@ -2747,7 +3143,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) assert freqtrade.handle_trade(trade) is False # Price fell - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': 0.00001099 * 1.1, 'ask': 0.00001099 * 1.1, @@ -2757,10 +3153,8 @@ def test_trailing_stop_loss(default_conf, limit_buy_order, fee, caplog, mocker) caplog.set_level(logging.DEBUG) # Sell as trailing-stop is reached assert freqtrade.handle_trade(trade) is True - assert log_has( - f"ETH/BTC - HIT STOP: current price at 0.000012, " - f"stoploss is 0.000015, " - f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) + assert log_has("ETH/BTC - HIT STOP: current price at 0.000012, stoploss is 0.000015, " + "initial stoploss was at 0.000010, trade opened at 0.000011", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value @@ -2771,7 +3165,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price - 0.000001, 'ask': buy_price - 0.000001, 'last': buy_price - 0.000001 @@ -2786,7 +3180,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2795,7 +3189,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, assert freqtrade.handle_trade(trade) is False # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.000003, 'ask': buy_price + 0.000003, @@ -2803,11 +3197,11 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, fee, })) # stop-loss not reached, adjusted stoploss assert freqtrade.handle_trade(trade) is False - assert log_has(f"ETH/BTC - Using positive stoploss: 0.01 offset: 0 profit: 0.2666%", caplog) - assert log_has(f"ETH/BTC - Adjusting stoploss...", caplog) + assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0 profit: 0.2666%", caplog) + assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.000002, 'ask': buy_price + 0.000002, @@ -2828,7 +3222,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price - 0.000001, 'ask': buy_price - 0.000001, 'last': buy_price - 0.000001 @@ -2843,7 +3237,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2852,7 +3246,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, assert freqtrade.handle_trade(trade) is False # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.000003, 'ask': buy_price + 0.000003, @@ -2860,12 +3254,11 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, fee, })) # stop-loss not reached, adjusted stoploss assert freqtrade.handle_trade(trade) is False - assert log_has(f"ETH/BTC - Using positive stoploss: 0.01 offset: 0.011 profit: 0.2666%", - caplog) - assert log_has(f"ETH/BTC - Adjusting stoploss...", caplog) + assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0.011 profit: 0.2666%", caplog) + assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.000002, 'ask': buy_price + 0.000002, @@ -2889,7 +3282,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': buy_price, 'ask': buy_price, 'last': buy_price @@ -2906,7 +3299,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=False) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2916,7 +3309,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, assert trade.stop_loss == 0.0000098910 # Raise ticker above buy price - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.0000004, 'ask': buy_price + 0.0000004, @@ -2926,11 +3319,11 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, # stop-loss should not be adjusted as offset is not reached yet assert freqtrade.handle_trade(trade) is False - assert not log_has(f"ETH/BTC - Adjusting stoploss...", caplog) + assert not log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000098910 # price rises above the offset (rises 12% when the offset is 5.5%) - mocker.patch('freqtrade.exchange.Exchange.get_ticker', + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ 'bid': buy_price + 0.0000014, 'ask': buy_price + 0.0000014, @@ -2938,9 +3331,8 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, fee, })) assert freqtrade.handle_trade(trade) is False - assert log_has(f"ETH/BTC - Using positive stoploss: 0.05 offset: 0.055 profit: 0.1218%", - caplog) - assert log_has(f"ETH/BTC - Adjusting stoploss...", caplog) + assert log_has("ETH/BTC - Using positive stoploss: 0.05 offset: 0.055 profit: 0.1218%", caplog) + assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000117705 @@ -2950,7 +3342,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00000172, 'ask': 0.00000173, 'last': 0.00000172 @@ -2965,7 +3357,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, patch_get_signal(freqtrade) freqtrade.strategy.min_roi_reached = MagicMock(return_value=True) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() trade.update(limit_buy_order) @@ -2981,8 +3373,6 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) - patch_RPCManager(mocker) - patch_exchange(mocker) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( pair='LTC/ETH', @@ -2993,21 +3383,43 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades', + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) +def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, + caplog, mocker): + mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) + walletmock = mocker.patch('freqtrade.wallets.Wallets.update') + mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=8.1122) + amount = sum(x['amount'] for x in trades_for_order) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_order_id="123456" + ) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + walletmock.reset_mock() + # Amount is kept as is + assert freqtrade.get_real_amount(trade, buy_order_fee) == amount + assert walletmock.call_count == 1 + assert log_has_re(r'Fee amount for Trade.* was in base currency ' + '- Eating Fee 0.008 into dust', caplog) + + def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) - patch_RPCManager(mocker) - patch_exchange(mocker) amount = buy_order_fee['amount'] trade = Trade( pair='LTC/ETH', @@ -3018,8 +3430,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -3031,8 +3442,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): trades_for_order[0]['fee']['currency'] = 'ETH' - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3044,8 +3453,7 @@ def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fe open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount @@ -3058,8 +3466,6 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ limit_buy_order['fee'] = {'cost': 0.004, 'currency': None} trades_for_order[0]['fee']['currency'] = None - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3071,8 +3477,7 @@ def test_get_real_amount_no_currency_in_fee(default_conf, trades_for_order, buy_ open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -3082,8 +3487,6 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, trades_for_order[0]['fee']['currency'] = 'BNB' trades_for_order[0]['fee']['cost'] = 0.00094518 - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3095,16 +3498,13 @@ def test_get_real_amount_BNB(default_conf, trades_for_order, buy_order_fee, fee, open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, caplog, fee, mocker): - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order2) amount = float(sum(x['amount'] for x in trades_for_order2)) trade = Trade( @@ -3116,13 +3516,12 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.001) assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992) from Trades', + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) @@ -3131,8 +3530,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004, 'currency': 'LTC'} - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[trades_for_order]) amount = float(sum(x['amount'] for x in trades_for_order)) @@ -3145,13 +3542,12 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount is reduced by "fee" assert freqtrade.get_real_amount(trade, limit_buy_order) == amount - 0.004 assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' - 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996) from Order', + 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', caplog) @@ -3159,8 +3555,6 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['fee'] = {'cost': 0.004} - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=[]) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -3172,8 +3566,7 @@ def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, limit_buy_order) == amount @@ -3183,8 +3576,6 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ limit_buy_order = deepcopy(buy_order_fee) limit_buy_order['amount'] = limit_buy_order['amount'] - 0.001 - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -3196,8 +3587,7 @@ def test_get_real_amount_wrong_amount(default_conf, trades_for_order, buy_order_ fee_close=fee.return_value, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change with pytest.raises(DependencyException, match=r"Half bought\? Amounts don't match"): @@ -3210,8 +3600,6 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b limit_buy_order = deepcopy(buy_order_fee) trades_for_order[0]['amount'] = trades_for_order[0]['amount'] + 1e-15 - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = float(sum(x['amount'] for x in trades_for_order)) trade = Trade( @@ -3223,8 +3611,7 @@ def test_get_real_amount_wrong_amount_rounding(default_conf, trades_for_order, b open_rate=0.245441, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount changes by fee amount. assert isclose(freqtrade.get_real_amount(trade, limit_buy_order), amount - (amount * 0.001), @@ -3235,8 +3622,6 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, # Remove "Currency" from fee dict trades_for_order[0]['fee'] = {'cost': 0.008} - patch_RPCManager(mocker) - patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order) amount = sum(x['amount'] for x in trades_for_order) trade = Trade( @@ -3249,15 +3634,12 @@ def test_get_real_amount_invalid(default_conf, trades_for_order, buy_order_fee, open_order_id="123456" ) - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) # Amount does not change assert freqtrade.get_real_amount(trade, buy_order_fee) == amount def test_get_real_amount_open_trade(default_conf, fee, mocker): - patch_RPCManager(mocker) - patch_exchange(mocker) amount = 12345 trade = Trade( pair='LTC/ETH', @@ -3272,12 +3654,41 @@ def test_get_real_amount_open_trade(default_conf, fee, mocker): 'id': 'mocked_order', 'amount': amount, 'status': 'open', + 'side': 'buy', } - freqtrade = FreqtradeBot(default_conf) - patch_get_signal(freqtrade) + freqtrade = get_patched_freqtradebot(mocker, default_conf) assert freqtrade.get_real_amount(trade, order) == amount +@pytest.mark.parametrize('amount,fee_abs,wallet,amount_exp', [ + (8.0, 0.0, 10, 8), + (8.0, 0.0, 0, 8), + (8.0, 0.1, 0, 7.9), + (8.0, 0.1, 10, 8), + (8.0, 0.1, 8.0, 8.0), + (8.0, 0.1, 7.9, 7.9), +]) +def test_apply_fee_conditional(default_conf, fee, caplog, mocker, + amount, fee_abs, wallet, amount_exp): + walletmock = mocker.patch('freqtrade.wallets.Wallets.update') + mocker.patch('freqtrade.wallets.Wallets.get_free', return_value=wallet) + trade = Trade( + pair='LTC/ETH', + amount=amount, + exchange='binance', + open_rate=0.245441, + fee_open=fee.return_value, + fee_close=fee.return_value, + open_order_id="123456" + ) + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + walletmock.reset_mock() + # Amount is kept as is + assert freqtrade.apply_fee_conditional(trade, 'LTC', amount, fee_abs) == amount_exp + assert walletmock.call_count == 1 + + def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker, order_book_l2): default_conf['bid_strategy']['check_depth_of_market']['enabled'] = True @@ -3287,7 +3698,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -3296,7 +3707,7 @@ def test_order_book_depth_of_market(default_conf, ticker, limit_buy_order, fee, whitelist = deepcopy(default_conf['exchange']['pair_whitelist']) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade is not None @@ -3324,14 +3735,14 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) # Save state of current whitelist freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade is None @@ -3339,7 +3750,7 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: """ - test if function get_target_bid will return the order book price + test if function get_buy_rate will return the order book price instead of the ask rate """ patch_exchange(mocker) @@ -3347,7 +3758,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_order_book=order_book_l2, - get_ticker=ticker_mock, + fetch_ticker=ticker_mock, ) default_conf['exchange']['name'] = 'binance' @@ -3357,13 +3768,13 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_target_bid('ETH/BTC') == 0.043935 + assert freqtrade.get_buy_rate('ETH/BTC', True) == 0.043935 assert ticker_mock.call_count == 0 def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: """ - test if function get_target_bid will return the ask rate (since its value is lower) + test if function get_buy_rate will return the ask rate (since its value is lower) instead of the order book rate (even if enabled) """ patch_exchange(mocker) @@ -3371,7 +3782,7 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_order_book=order_book_l2, - get_ticker=ticker_mock, + fetch_ticker=ticker_mock, ) default_conf['exchange']['name'] = 'binance' @@ -3382,7 +3793,7 @@ def test_order_book_bid_strategy2(mocker, default_conf, order_book_l2) -> None: freqtrade = FreqtradeBot(default_conf) # orderbook shall be used even if tickers would be lower. - assert freqtrade.get_target_bid('ETH/BTC') != 0.042 + assert freqtrade.get_buy_rate('ETH/BTC', True) != 0.042 assert ticker_mock.call_count == 0 @@ -3421,7 +3832,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=MagicMock(return_value={ + fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, 'last': 0.00001172 @@ -3433,42 +3844,69 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order, limit_sell_order freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - freqtrade.create_trades() + freqtrade.enter_positions() trade = Trade.query.first() assert trade time.sleep(0.01) # Race condition fix trade.update(limit_buy_order) + freqtrade.wallets.update() assert trade.is_open is True patch_get_signal(freqtrade, value=(False, True)) assert freqtrade.handle_trade(trade) is True -def test_get_sell_rate(default_conf, mocker, ticker, order_book_l2) -> None: - - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - get_order_book=order_book_l2, - get_ticker=ticker, - ) +@pytest.mark.parametrize('side,ask,bid,expected', [ + ('bid', 10.0, 11.0, 11.0), + ('bid', 10.0, 11.2, 11.2), + ('bid', 10.0, 11.0, 11.0), + ('bid', 9.8, 11.0, 11.0), + ('bid', 0.0001, 0.002, 0.002), + ('ask', 10.0, 11.0, 10.0), + ('ask', 10.11, 11.2, 10.11), + ('ask', 0.001, 0.002, 0.001), + ('ask', 0.006, 1.0, 0.006), +]) +def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, expected) -> None: + default_conf['ask_strategy']['price_side'] = side + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'ask': ask, 'bid': bid}) pair = "ETH/BTC" # Test regular mode ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) - assert rate == 0.00001098 + assert rate == expected + # Use caching + rate = ft.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + +@pytest.mark.parametrize('side,expected', [ + ('bid', 0.043936), # Value from order_book_l2 fiture - bids side + ('ask', 0.043949), # Value from order_book_l2 fiture - asks side +]) +def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2): # Test orderbook mode + default_conf['ask_strategy']['price_side'] = side default_conf['ask_strategy']['use_order_book'] = True default_conf['ask_strategy']['order_book_min'] = 1 default_conf['ask_strategy']['order_book_max'] = 2 + # TODO: min/max is irrelevant for this test until refactoring + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.get_order_book', order_book_l2) ft = get_patched_freqtradebot(mocker, default_conf) rate = ft.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) - assert rate == 0.043936 + assert rate == expected + rate = ft.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) def test_startup_state(default_conf, mocker): @@ -3477,7 +3915,7 @@ def test_startup_state(default_conf, mocker): } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) worker = get_patched_worker(mocker, default_conf) - assert worker.state is State.RUNNING + assert worker.freqtrade.state is State.RUNNING def test_startup_trade_reinit(default_conf, edge_conf, mocker): @@ -3497,40 +3935,17 @@ def test_startup_trade_reinit(default_conf, edge_conf, mocker): assert reinit_mock.call_count == 0 -def test_process_i_am_alive(default_conf, mocker, caplog): - patch_RPCManager(mocker) - patch_exchange(mocker) - mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) - - ftbot = get_patched_freqtradebot(mocker, default_conf) - message = r"Bot heartbeat\. PID=.*" - ftbot.process() - assert log_has_re(message, caplog) - assert ftbot._heartbeat_msg != 0 - - caplog.clear() - # Message is not shown before interval is up - ftbot.process() - assert not log_has_re(message, caplog) - - caplog.clear() - # Set clock - 70 seconds - ftbot._heartbeat_msg -= 70 - - ftbot.process() - assert log_has_re(message, caplog) - - @pytest.mark.usefixtures("init_persistence") -def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order): +def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order, caplog): default_conf['dry_run'] = True # Initialize to 2 times stake amount default_conf['dry_run_wallet'] = 0.002 default_conf['max_open_trades'] = 2 + default_conf['tradable_balance_ratio'] = 1.0 patch_exchange(mocker) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, buy=MagicMock(return_value={'id': limit_buy_order['id']}), get_fee=fee, ) @@ -3539,12 +3954,31 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order) patch_get_signal(bot) assert bot.wallets.get_free('BTC') == 0.002 - bot.create_trades() + n = bot.enter_positions() + assert n == 2 trades = Trade.query.all() assert len(trades) == 2 bot.config['max_open_trades'] = 3 - with pytest.raises( - DependencyException, - match=r"Available balance \(0 BTC\) is lower than stake amount \(0.001 BTC\)"): - bot.create_trades() + n = bot.enter_positions() + assert n == 0 + assert log_has_re(r"Unable to create trade for XRP/BTC: " + r"Available balance \(0.0 BTC\) is lower than stake amount \(0.001 BTC\)", + caplog) + + +@pytest.mark.usefixtures("init_persistence") +def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limit_sell_order): + default_conf['cancel_open_orders_on_exit'] = True + mocker.patch('freqtrade.exchange.Exchange.get_order', + side_effect=[DependencyException(), limit_sell_order, limit_buy_order]) + buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_buy') + sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_sell') + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + create_mock_trades(fee) + trades = Trade.query.all() + assert len(trades) == 3 + freqtrade.cancel_all_open_orders() + assert buy_mock.call_count == 1 + assert sell_mock.call_count == 1 diff --git a/tests/test_integration.py b/tests/test_integration.py index 728e96d55..1396e86f5 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,10 +1,11 @@ - from unittest.mock import MagicMock +import pytest + from freqtrade.persistence import Trade +from freqtrade.rpc.rpc import RPC from freqtrade.strategy.interface import SellCheckTuple, SellType from tests.conftest import get_patched_freqtradebot, patch_get_signal -from freqtrade.rpc.rpc import RPC def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, @@ -19,7 +20,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, default_conf['max_open_trades'] = 3 default_conf['exchange']['name'] = 'binance' - stoploss_limit = { + stoploss = { 'id': 123, 'info': {} } @@ -43,6 +44,8 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, } stoploss_order_closed = stoploss_order_open.copy() stoploss_order_closed['status'] = 'closed' + stoploss_order_closed['filled'] = stoploss_order_closed['amount'] + # Sell first trade based on stoploss, keep 2nd and 3rd trade open stoploss_order_mock = MagicMock( side_effect=[stoploss_order_closed, stoploss_order_open, stoploss_order_open]) @@ -52,13 +55,13 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL)] ) cancel_order_mock = MagicMock() - mocker.patch('freqtrade.exchange.Binance.stoploss_limit', stoploss_limit) + mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, - symbol_amount_prec=lambda s, x, y: y, - symbol_price_prec=lambda s, x, y: y, + amount_to_precision=lambda s, x, y: y, + price_to_precision=lambda s, x, y: y, get_order=stoploss_order_mock, cancel_order=cancel_order_mock, ) @@ -66,12 +69,11 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - update_trade_state=MagicMock(), _notify_sell=MagicMock(), ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update", MagicMock()) - mocker.patch("freqtrade.wallets.Wallets.get_free", MagicMock(return_value=1)) + mocker.patch("freqtrade.wallets.Wallets.get_free", MagicMock(return_value=1000)) freqtrade = get_patched_freqtradebot(mocker, default_conf) freqtrade.strategy.order_types['stoploss_on_exchange'] = True @@ -80,7 +82,7 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, patch_get_signal(freqtrade) # Create some test data - freqtrade.create_trades() + freqtrade.enter_positions() wallets_mock.reset_mock() Trade.session = MagicMock() @@ -90,13 +92,15 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, trade.stoploss_order_id = 3 trade.open_order_id = None - freqtrade.process_maybe_execute_sells(trades) + n = freqtrade.exit_positions(trades) + assert n == 2 assert should_sell_mock.call_count == 2 # Only order for 3rd trade needs to be cancelled assert cancel_order_mock.call_count == 1 - # Wallets should only be called once per sell cycle - assert wallets_mock.call_count == 1 + # Wallets must be updated between stoploss cancellation and selling, and will be updated again + # during update_trade_state + assert wallets_mock.call_count == 4 trade = trades[0] assert trade.sell_reason == SellType.STOPLOSS_ON_EXCHANGE.value @@ -111,33 +115,47 @@ def test_may_execute_sell_stoploss_on_exchange_multi(default_conf, ticker, fee, assert not trade.is_open -def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, mocker) -> None: +@pytest.mark.parametrize("balance_ratio,result1", [ + (1, 200), + (0.99, 198), +]) +def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, mocker, balance_ratio, + result1) -> None: """ - Tests workflow + Tests workflow unlimited stake-amount + Buy 4 trades, forcebuy a 5th trade + Sell one trade, calculated stake amount should now be lower than before since + one trade was sold at a loss. """ default_conf['max_open_trades'] = 5 default_conf['forcebuy_enable'] = True default_conf['stake_amount'] = 'unlimited' + default_conf['tradable_balance_ratio'] = balance_ratio + default_conf['dry_run_wallet'] = 1000 default_conf['exchange']['name'] = 'binance' default_conf['telegram']['enabled'] = True mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock()) - mocker.patch('freqtrade.wallets.Wallets.get_free', MagicMock( - side_effect=[1000, 800, 600, 400, 200] - )) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_ticker=ticker, + fetch_ticker=ticker, get_fee=fee, - symbol_amount_prec=lambda s, x, y: y, - symbol_price_prec=lambda s, x, y: y, + amount_to_precision=lambda s, x, y: y, + price_to_precision=lambda s, x, y: y, ) mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - update_trade_state=MagicMock(), _notify_sell=MagicMock(), ) + should_sell_mock = MagicMock(side_effect=[ + SellCheckTuple(sell_flag=False, sell_type=SellType.NONE), + SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL), + SellCheckTuple(sell_flag=False, sell_type=SellType.NONE), + SellCheckTuple(sell_flag=False, sell_type=SellType.NONE), + SellCheckTuple(sell_flag=None, sell_type=SellType.NONE)] + ) + mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) freqtrade = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtrade) @@ -147,14 +165,37 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc patch_get_signal(freqtrade) # Create 4 trades - freqtrade.create_trades() + n = freqtrade.enter_positions() + assert n == 4 trades = Trade.query.all() assert len(trades) == 4 + assert freqtrade.get_trade_stake_amount('XRP/BTC') == result1 + rpc._rpc_forcebuy('TKN/BTC', None) trades = Trade.query.all() assert len(trades) == 5 for trade in trades: - assert trade.stake_amount == 200 + assert trade.stake_amount == result1 + # Reset trade open order id's + trade.open_order_id = None + trades = Trade.get_open_trades() + assert len(trades) == 5 + bals = freqtrade.wallets.get_all_balances() + + n = freqtrade.exit_positions(trades) + assert n == 1 + trades = Trade.get_open_trades() + # One trade sold + assert len(trades) == 4 + # stake-amount should now be reduced, since one trade was sold at a loss. + assert freqtrade.get_trade_stake_amount('XRP/BTC') < result1 + # Validate that balance of sold trade is not in dry-run balances anymore. + bals2 = freqtrade.wallets.get_all_balances() + assert bals != bals2 + assert len(bals) == 6 + assert len(bals2) == 5 + assert 'LTC' in bals + assert 'LTC' not in bals2 diff --git a/tests/test_main.py b/tests/test_main.py index 03e6a7ce9..11d0ede3a 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -5,8 +5,9 @@ from unittest.mock import MagicMock, PropertyMock import pytest -from freqtrade import OperationalException -from freqtrade.configuration import Arguments +from pathlib import Path +from freqtrade.commands import Arguments +from freqtrade.exceptions import FreqtradeException, OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main from freqtrade.state import State @@ -26,7 +27,8 @@ def test_parse_args_backtesting(mocker) -> None: Test that main() can start backtesting and also ensure we can pass some specific arguments further argument parsing is done in test_arguments.py """ - backtesting_mock = mocker.patch('freqtrade.optimize.start_backtesting', MagicMock()) + mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True])) + backtesting_mock = mocker.patch('freqtrade.commands.start_backtesting') backtesting_mock.__name__ = PropertyMock("start_backtesting") # it's sys.exit(0) at the end of backtesting with pytest.raises(SystemExit): @@ -42,7 +44,8 @@ def test_parse_args_backtesting(mocker) -> None: def test_main_start_hyperopt(mocker) -> None: - hyperopt_mock = mocker.patch('freqtrade.optimize.start_hyperopt', MagicMock()) + mocker.patch.object(Path, "is_file", MagicMock(side_effect=[False, True])) + hyperopt_mock = mocker.patch('freqtrade.commands.start_hyperopt', MagicMock()) hyperopt_mock.__name__ = PropertyMock("start_hyperopt") # it's sys.exit(0) at the end of hyperopt with pytest.raises(SystemExit): @@ -96,7 +99,7 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cleanup', MagicMock()) mocker.patch( 'freqtrade.worker.Worker._worker', - MagicMock(side_effect=OperationalException('Oh snap!')) + MagicMock(side_effect=FreqtradeException('Oh snap!')) ) patched_configuration_load_config_file(mocker, default_conf) mocker.patch('freqtrade.wallets.Wallets.update', MagicMock()) @@ -112,6 +115,32 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: assert log_has('Oh snap!', caplog) +def test_main_operational_exception1(mocker, default_conf, caplog) -> None: + patch_exchange(mocker) + mocker.patch( + 'freqtrade.commands.list_commands.available_exchanges', + MagicMock(side_effect=ValueError('Oh snap!')) + ) + patched_configuration_load_config_file(mocker, default_conf) + + args = ['list-exchanges'] + + # Test Main + the KeyboardInterrupt exception + with pytest.raises(SystemExit): + main(args) + + assert log_has('Fatal exception!', caplog) + assert not log_has_re(r'SIGINT.*', caplog) + mocker.patch( + 'freqtrade.commands.list_commands.available_exchanges', + MagicMock(side_effect=KeyboardInterrupt) + ) + with pytest.raises(SystemExit): + main(args) + + assert log_has_re(r'SIGINT.*', caplog) + + def test_main_reload_conf(mocker, default_conf, caplog) -> None: patch_exchange(mocker) mocker.patch('freqtrade.freqtradebot.FreqtradeBot.cleanup', MagicMock()) diff --git a/tests/test_misc.py b/tests/test_misc.py index 23231e2f0..9fd6164d5 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -4,10 +4,14 @@ import datetime from pathlib import Path from unittest.mock import MagicMock -from freqtrade.data.converter import parse_ticker_dataframe -from freqtrade.data.history import pair_data_filename +import pytest + +from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.misc import (datesarray_to_datetimearray, file_dump_json, - file_load_json, format_ms_time, plural, shorten_date) + file_load_json, format_ms_time, pair_to_filename, + plural, render_template, + render_template_with_fallback, safe_value_fallback, + shorten_date) def test_shorten_date() -> None: @@ -16,9 +20,9 @@ def test_shorten_date() -> None: assert shorten_date(str_data) == str_shorten_data -def test_datesarray_to_datetimearray(ticker_history_list): - dataframes = parse_ticker_dataframe(ticker_history_list, "5m", pair="UNITTEST/BTC", - fill_missing=True) +def test_datesarray_to_datetimearray(ohlcv_history_list): + dataframes = ohlcv_to_dataframe(ohlcv_history_list, "5m", pair="UNITTEST/BTC", + fill_missing=True) dates = datesarray_to_datetimearray(dataframes['date']) assert isinstance(dates[0], datetime.datetime) @@ -48,16 +52,36 @@ def test_file_dump_json(mocker) -> None: def test_file_load_json(mocker, testdatadir) -> None: # 7m .json does not exist - ret = file_load_json(pair_data_filename(testdatadir, 'UNITTEST/BTC', '7m')) + ret = file_load_json(testdatadir / 'UNITTEST_BTC-7m.json') assert not ret # 1m json exists (but no .gz exists) - ret = file_load_json(pair_data_filename(testdatadir, 'UNITTEST/BTC', '1m')) + ret = file_load_json(testdatadir / 'UNITTEST_BTC-1m.json') assert ret # 8 .json is empty and will fail if it's loaded. .json.gz is a copy of 1.json - ret = file_load_json(pair_data_filename(testdatadir, 'UNITTEST/BTC', '8m')) + ret = file_load_json(testdatadir / 'UNITTEST_BTC-8m.json') assert ret +@pytest.mark.parametrize("pair,expected_result", [ + ("ETH/BTC", 'ETH_BTC'), + ("Fabric Token/ETH", 'Fabric_Token_ETH'), + ("ETHH20", 'ETHH20'), + (".XBTBON2H", '_XBTBON2H'), + ("ETHUSD.d", 'ETHUSD_d'), + ("ADA-0327", 'ADA_0327'), + ("BTC-USD-200110", 'BTC_USD_200110'), + ("F-AKRO/USDT", 'F_AKRO_USDT'), + ("LC+/ETH", 'LC__ETH'), + ("CMT@18/ETH", 'CMT_18_ETH'), + ("LBTC:1022/SAI", 'LBTC_1022_SAI'), + ("$PAC/BTC", '_PAC_BTC'), + ("ACC_OLD/BTC", 'ACC_OLD_BTC'), +]) +def test_pair_to_filename(pair, expected_result): + pair_s = pair_to_filename(pair) + assert pair_s == expected_result + + def test_format_ms_time() -> None: # Date 2018-04-10 18:02:01 date_in_epoch_ms = 1523383321000 @@ -71,6 +95,27 @@ def test_format_ms_time() -> None: assert format_ms_time(date_in_epoch_ms) == res.astimezone(None).strftime('%Y-%m-%dT%H:%M:%S') +def test_safe_value_fallback(): + dict1 = {'keya': None, 'keyb': 2, 'keyc': 5, 'keyd': None} + dict2 = {'keya': 20, 'keyb': None, 'keyc': 6, 'keyd': None} + assert safe_value_fallback(dict1, dict2, 'keya', 'keya') == 20 + assert safe_value_fallback(dict2, dict1, 'keya', 'keya') == 20 + + assert safe_value_fallback(dict1, dict2, 'keyb', 'keyb') == 2 + assert safe_value_fallback(dict2, dict1, 'keyb', 'keyb') == 2 + + assert safe_value_fallback(dict1, dict2, 'keyc', 'keyc') == 5 + assert safe_value_fallback(dict2, dict1, 'keyc', 'keyc') == 6 + + assert safe_value_fallback(dict1, dict2, 'keyd', 'keyd') is None + assert safe_value_fallback(dict2, dict1, 'keyd', 'keyd') is None + assert safe_value_fallback(dict2, dict1, 'keyd', 'keyd', 1234) == 1234 + + assert safe_value_fallback(dict1, dict2, 'keyNo', 'keyNo') is None + assert safe_value_fallback(dict2, dict1, 'keyNo', 'keyNo') is None + assert safe_value_fallback(dict2, dict1, 'keyNo', 'keyNo', 1234) == 1234 + + def test_plural() -> None: assert plural(0, "page") == "pages" assert plural(0.0, "page") == "pages" @@ -101,3 +146,17 @@ def test_plural() -> None: assert plural(1.5, "ox", "oxen") == "oxen" assert plural(-0.5, "ox", "oxen") == "oxen" assert plural(-1.5, "ox", "oxen") == "oxen" + + +def test_render_template_fallback(mocker): + from jinja2.exceptions import TemplateNotFound + with pytest.raises(TemplateNotFound): + val = render_template( + templatefile='subtemplates/indicators_does-not-exist.j2',) + + val = render_template_with_fallback( + templatefile='subtemplates/indicators_does-not-exist.j2', + templatefallbackfile='subtemplates/indicators_minimal.j2', + ) + assert isinstance(val, str) + assert 'if self.dp' in val diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 25ad8b6a7..25afed397 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -6,55 +6,10 @@ import arrow import pytest from sqlalchemy import create_engine -from freqtrade import OperationalException, constants +from freqtrade import constants +from freqtrade.exceptions import OperationalException from freqtrade.persistence import Trade, clean_dry_run_db, init -from tests.conftest import log_has - - -def create_mock_trades(fee): - """ - Create some fake trades ... - """ - # Simulate dry_run entries - trade = Trade( - pair='ETH/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='dry_run_buy_12345' - ) - Trade.session.add(trade) - - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - close_rate=0.128, - close_profit=0.005, - exchange='bittrex', - is_open=False, - open_order_id='dry_run_sell_12345' - ) - Trade.session.add(trade) - - # Simulate prod entry - trade = Trade( - pair='ETC/BTC', - stake_amount=0.001, - amount=123.0, - fee_open=fee.return_value, - fee_close=fee.return_value, - open_rate=0.123, - exchange='bittrex', - open_order_id='prod_buy_12345' - ) - Trade.session.add(trade) +from tests.conftest import log_has, create_mock_trades def test_init_create_session(default_conf): @@ -100,7 +55,7 @@ def test_init_dryrun_db(default_conf, mocker): init(default_conf['db_url'], default_conf['dry_run']) assert create_engine_mock.call_count == 1 - assert create_engine_mock.mock_calls[0][1][0] == 'sqlite://' + assert create_engine_mock.mock_calls[0][1][0] == 'sqlite:///tradesv3.dryrun.sqlite' @pytest.mark.usefixtures("init_persistence") @@ -475,12 +430,22 @@ def test_migrate_old(mocker, default_conf, fee): stake=default_conf.get("stake_amount"), amount=amount ) + insert_table_old2 = """INSERT INTO trades (exchange, pair, is_open, fee, + open_rate, close_rate, stake_amount, amount, open_date) + VALUES ('BITTREX', 'BTC_ETC', 0, {fee}, + 0.00258580, 0.00268580, {stake}, {amount}, + '2017-11-28 12:44:24.000000') + """.format(fee=fee.return_value, + stake=default_conf.get("stake_amount"), + amount=amount + ) engine = create_engine('sqlite://') mocker.patch('freqtrade.persistence.create_engine', lambda *args, **kwargs: engine) # Create table using the old format engine.execute(create_table_old) engine.execute(insert_table_old) + engine.execute(insert_table_old2) # Run init to test migration init(default_conf['db_url'], default_conf['dry_run']) @@ -499,6 +464,20 @@ def test_migrate_old(mocker, default_conf, fee): assert trade.stop_loss == 0.0 assert trade.initial_stop_loss == 0.0 assert trade.open_trade_price == trade._calc_open_trade_price() + assert trade.close_profit_abs is None + assert trade.fee_open_cost is None + assert trade.fee_open_currency is None + assert trade.fee_close_cost is None + assert trade.fee_close_currency is None + + trade = Trade.query.filter(Trade.id == 2).first() + assert trade.close_rate is not None + assert trade.is_open == 0 + assert trade.open_rate_requested is None + assert trade.close_rate_requested is None + assert trade.close_rate is not None + assert pytest.approx(trade.close_profit_abs) == trade.calc_profit() + assert trade.sell_order_status is None def test_migrate_new(mocker, default_conf, fee, caplog): @@ -582,6 +561,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog): assert log_has("trying trades_bak2", caplog) assert log_has("Running database migration - backup available as trades_bak2", caplog) assert trade.open_trade_price == trade._calc_open_trade_price() + assert trade.close_profit_abs is None def test_migrate_mid_state(mocker, default_conf, fee, caplog): @@ -756,18 +736,36 @@ def test_to_json(default_conf, fee): assert result == {'trade_id': None, 'pair': 'ETH/BTC', + 'is_open': None, 'open_date_hum': '2 hours ago', 'open_date': trade.open_date.strftime("%Y-%m-%d %H:%M:%S"), + 'open_order_id': 'dry_run_buy_12345', 'close_date_hum': None, 'close_date': None, 'open_rate': 0.123, + 'open_rate_requested': None, + 'open_trade_price': 15.1668225, + 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, + 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, 'close_rate': None, + 'close_rate_requested': None, 'amount': 123.0, 'stake_amount': 0.001, + 'close_profit': None, + 'sell_reason': None, + 'sell_order_status': None, 'stop_loss': None, 'stop_loss_pct': None, 'initial_stop_loss': None, - 'initial_stop_loss_pct': None} + 'initial_stop_loss_pct': None, + 'min_rate': None, + 'max_rate': None, + 'strategy': None, + 'ticker_interval': None} # Simulate dry_run entries trade = Trade( @@ -798,7 +796,25 @@ def test_to_json(default_conf, fee): 'stop_loss': None, 'stop_loss_pct': None, 'initial_stop_loss': None, - 'initial_stop_loss_pct': None} + 'initial_stop_loss_pct': None, + 'close_profit': None, + 'close_rate_requested': None, + 'fee_close': 0.0025, + 'fee_close_cost': None, + 'fee_close_currency': None, + 'fee_open': 0.0025, + 'fee_open_cost': None, + 'fee_open_currency': None, + 'is_open': None, + 'max_rate': None, + 'min_rate': None, + 'open_order_id': None, + 'open_rate_requested': None, + 'open_trade_price': 12.33075, + 'sell_reason': None, + 'sell_order_status': None, + 'strategy': None, + 'ticker_interval': None} def test_stoploss_reinitialization(default_conf, fee): @@ -861,6 +877,75 @@ def test_stoploss_reinitialization(default_conf, fee): assert trade_adj.initial_stop_loss_pct == -0.04 +def test_update_fee(fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + open_date=arrow.utcnow().shift(hours=-2).datetime, + amount=10, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + max_rate=1, + ) + fee_cost = 0.15 + fee_currency = 'BTC' + fee_rate = 0.0075 + assert trade.fee_open_currency is None + assert not trade.fee_updated('buy') + assert not trade.fee_updated('sell') + + trade.update_fee(fee_cost, fee_currency, fee_rate, 'buy') + assert trade.fee_updated('buy') + assert not trade.fee_updated('sell') + assert trade.fee_open_currency == fee_currency + assert trade.fee_open_cost == fee_cost + assert trade.fee_open == fee_rate + # Setting buy rate should "guess" close rate + assert trade.fee_close == fee_rate + assert trade.fee_close_currency is None + assert trade.fee_close_cost is None + + fee_rate = 0.0076 + trade.update_fee(fee_cost, fee_currency, fee_rate, 'sell') + assert trade.fee_updated('buy') + assert trade.fee_updated('sell') + assert trade.fee_close == 0.0076 + assert trade.fee_close_cost == fee_cost + assert trade.fee_close == fee_rate + + +def test_fee_updated(fee): + trade = Trade( + pair='ETH/BTC', + stake_amount=0.001, + fee_open=fee.return_value, + open_date=arrow.utcnow().shift(hours=-2).datetime, + amount=10, + fee_close=fee.return_value, + exchange='bittrex', + open_rate=1, + max_rate=1, + ) + + assert trade.fee_open_currency is None + assert not trade.fee_updated('buy') + assert not trade.fee_updated('sell') + assert not trade.fee_updated('asdf') + + trade.update_fee(0.15, 'BTC', 0.0075, 'buy') + assert trade.fee_updated('buy') + assert not trade.fee_updated('sell') + assert trade.fee_open_currency is not None + assert trade.fee_close_currency is None + + trade.update_fee(0.15, 'ABC', 0.0075, 'sell') + assert trade.fee_updated('buy') + assert trade.fee_updated('sell') + assert not trade.fee_updated('asfd') + + @pytest.mark.usefixtures("init_persistence") def test_total_open_trades_stakes(fee): diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 31502cafc..0258b94d1 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -3,22 +3,24 @@ from copy import deepcopy from pathlib import Path from unittest.mock import MagicMock +import pandas as pd import plotly.graph_objects as go import pytest from plotly.subplots import make_subplots -from freqtrade import OperationalException +from freqtrade.commands import start_plot_dataframe, start_plot_profit from freqtrade.configuration import TimeRange from freqtrade.data import history from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data -from freqtrade.plot.plot_utils import start_plot_dataframe, start_plot_profit +from freqtrade.exceptions import OperationalException from freqtrade.plot.plotting import (add_indicators, add_profit, - load_and_plot_trades, + create_plotconfig, generate_candlestick_graph, generate_plot_filename, generate_profit_graph, init_plotscript, - plot_profit, plot_trades, store_plot_file) -from freqtrade.strategy.default_strategy import DefaultStrategy + load_and_plot_trades, plot_profit, + plot_trades, store_plot_file) +from freqtrade.resolvers import StrategyResolver from tests.conftest import get_args, log_has, log_has_re @@ -47,17 +49,17 @@ def test_init_plotscript(default_conf, mocker, testdatadir): default_conf['trade_source'] = "file" default_conf['ticker_interval'] = "5m" default_conf["datadir"] = testdatadir - default_conf['exportfilename'] = str(testdatadir / "backtest-result_test.json") + default_conf['exportfilename'] = testdatadir / "backtest-result_test.json" ret = init_plotscript(default_conf) - assert "tickers" in ret + assert "ohlcv" in ret assert "trades" in ret assert "pairs" in ret default_conf['pairs'] = ["TRX/BTC", "ADA/BTC"] ret = init_plotscript(default_conf) - assert "tickers" in ret - assert "TRX/BTC" in ret["tickers"] - assert "ADA/BTC" in ret["tickers"] + assert "ohlcv" in ret + assert "TRX/BTC" in ret["ohlcv"] + assert "ADA/BTC" in ret["ohlcv"] def test_add_indicators(default_conf, testdatadir, caplog): @@ -66,12 +68,14 @@ def test_add_indicators(default_conf, testdatadir, caplog): data = history.load_pair_history(pair=pair, timeframe='1m', datadir=testdatadir, timerange=timerange) - indicators1 = ["ema10"] - indicators2 = ["macd"] + indicators1 = {"ema10": {}} + indicators2 = {"macd": {"color": "red"}} + + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) # Generate buy/sell signals and indicators - strat = DefaultStrategy(default_conf) - data = strat.analyze_ticker(data, {'pair': pair}) + data = strategy.analyze_ticker(data, {'pair': pair}) fig = generate_empty_figure() # Row 1 @@ -86,9 +90,10 @@ def test_add_indicators(default_conf, testdatadir, caplog): macd = find_trace_in_fig_data(figure.data, "macd") assert isinstance(macd, go.Scatter) assert macd.yaxis == "y3" + assert macd.line.color == "red" # No indicator found - fig3 = add_indicators(fig=deepcopy(fig), row=3, indicators=['no_indicator'], data=data) + fig3 = add_indicators(fig=deepcopy(fig), row=3, indicators={'no_indicator': {}}, data=data) assert fig == fig3 assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog) @@ -108,17 +113,29 @@ def test_plot_trades(testdatadir, caplog): figure = fig1.layout.figure # Check buys - color, should be in first graph, ... - trade_buy = find_trace_in_fig_data(figure.data, "trade_buy") + trade_buy = find_trace_in_fig_data(figure.data, 'Trade buy') assert isinstance(trade_buy, go.Scatter) assert trade_buy.yaxis == 'y' assert len(trades) == len(trade_buy.x) - assert trade_buy.marker.color == 'green' + assert trade_buy.marker.color == 'cyan' + assert trade_buy.marker.symbol == 'circle-open' + assert trade_buy.text[0] == '4.0%, roi, 15 min' - trade_sell = find_trace_in_fig_data(figure.data, "trade_sell") + trade_sell = find_trace_in_fig_data(figure.data, 'Sell - Profit') assert isinstance(trade_sell, go.Scatter) assert trade_sell.yaxis == 'y' - assert len(trades) == len(trade_sell.x) - assert trade_sell.marker.color == 'red' + assert len(trades.loc[trades['profitperc'] > 0]) == len(trade_sell.x) + assert trade_sell.marker.color == 'green' + assert trade_sell.marker.symbol == 'square-open' + assert trade_sell.text[0] == '4.0%, roi, 15 min' + + trade_sell_loss = find_trace_in_fig_data(figure.data, 'Sell - Loss') + assert isinstance(trade_sell_loss, go.Scatter) + assert trade_sell_loss.yaxis == 'y' + assert len(trades.loc[trades['profitperc'] <= 0]) == len(trade_sell_loss.x) + assert trade_sell_loss.marker.color == 'red' + assert trade_sell_loss.marker.symbol == 'square-open' + assert trade_sell_loss.text[5] == '-10.4%, stop_loss, 720 min' def test_generate_candlestick_graph_no_signals_no_trades(default_conf, mocker, testdatadir, caplog): @@ -167,9 +184,11 @@ def test_generate_candlestick_graph_no_trades(default_conf, mocker, testdatadir) data = history.load_pair_history(pair=pair, timeframe='1m', datadir=testdatadir, timerange=timerange) + default_conf.update({'strategy': 'DefaultStrategy'}) + strategy = StrategyResolver.load_strategy(default_conf) + # Generate buy/sell signals and indicators - strat = DefaultStrategy(default_conf) - data = strat.analyze_ticker(data, {'pair': pair}) + data = strategy.analyze_ticker(data, {'pair': pair}) indicators1 = [] indicators2 = [] @@ -247,16 +266,17 @@ def test_generate_profit_graph(testdatadir): filename = testdatadir / "backtest-result_test.json" trades = load_backtest_data(filename) timerange = TimeRange.parse_timerange("20180110-20180112") - pairs = ["TRX/BTC", "ADA/BTC"] + pairs = ["TRX/BTC", "XLM/BTC"] + trades = trades[trades['close_time'] < pd.Timestamp('2018-01-12', tz='UTC')] + + data = history.load_data(datadir=testdatadir, + pairs=pairs, + timeframe='5m', + timerange=timerange) - tickers = history.load_data(datadir=testdatadir, - pairs=pairs, - timeframe='5m', - timerange=timerange - ) trades = trades[trades['pair'].isin(pairs)] - fig = generate_profit_graph(pairs, tickers, trades, timeframe="5m") + fig = generate_profit_graph(pairs, data, trades, timeframe="5m") assert isinstance(fig, go.Figure) assert fig.layout.title.text == "Freqtrade Profit plot" @@ -265,13 +285,15 @@ def test_generate_profit_graph(testdatadir): assert fig.layout.yaxis3.title.text == "Profit" figure = fig.layout.figure - assert len(figure.data) == 4 + assert len(figure.data) == 5 avgclose = find_trace_in_fig_data(figure.data, "Avg close price") assert isinstance(avgclose, go.Scatter) profit = find_trace_in_fig_data(figure.data, "Profit") assert isinstance(profit, go.Scatter) + profit = find_trace_in_fig_data(figure.data, "Max drawdown 10.45%") + assert isinstance(profit, go.Scatter) for pair in pairs: profit_pair = find_trace_in_fig_data(figure.data, f"Profit {pair}") @@ -296,7 +318,7 @@ def test_start_plot_dataframe(mocker): def test_load_and_plot_trades(default_conf, mocker, caplog, testdatadir): default_conf['trade_source'] = 'file' default_conf["datadir"] = testdatadir - default_conf['exportfilename'] = str(testdatadir / "backtest-result_test.json") + default_conf['exportfilename'] = testdatadir / "backtest-result_test.json" default_conf['indicators1'] = ["sma5", "ema10"] default_conf['indicators2'] = ["macd"] default_conf['pairs'] = ["ETH/BTC", "LTC/BTC"] @@ -307,7 +329,7 @@ def test_load_and_plot_trades(default_conf, mocker, caplog, testdatadir): "freqtrade.plot.plotting", generate_candlestick_graph=candle_mock, store_plot_file=store_mock - ) + ) load_and_plot_trades(default_conf) # Both mocks should be called once per pair @@ -352,7 +374,7 @@ def test_start_plot_profit_error(mocker): def test_plot_profit(default_conf, mocker, testdatadir, caplog): default_conf['trade_source'] = 'file' default_conf["datadir"] = testdatadir - default_conf['exportfilename'] = str(testdatadir / "backtest-result_test.json") + default_conf['exportfilename'] = testdatadir / "backtest-result_test.json" default_conf['pairs'] = ["ETH/BTC", "LTC/BTC"] profit_mock = MagicMock() @@ -370,3 +392,47 @@ def test_plot_profit(default_conf, mocker, testdatadir, caplog): assert profit_mock.call_args_list[0][0][0] == default_conf['pairs'] assert store_mock.call_args_list[0][1]['auto_open'] is True + + +@pytest.mark.parametrize("ind1,ind2,plot_conf,exp", [ + # No indicators, use plot_conf + ([], [], {}, + {'main_plot': {'sma': {}, 'ema3': {}, 'ema5': {}}, + 'subplots': {'Other': {'macd': {}, 'macdsignal': {}}}}), + # use indicators + (['sma', 'ema3'], ['macd'], {}, + {'main_plot': {'sma': {}, 'ema3': {}}, 'subplots': {'Other': {'macd': {}}}}), + # only main_plot - adds empty subplots + ([], [], {'main_plot': {'sma': {}}}, + {'main_plot': {'sma': {}}, 'subplots': {}}), + # Main and subplots + ([], [], {'main_plot': {'sma': {}}, 'subplots': {'RSI': {'rsi': {'color': 'red'}}}}, + {'main_plot': {'sma': {}}, 'subplots': {'RSI': {'rsi': {'color': 'red'}}}}), + # no main_plot, adds empty main_plot + ([], [], {'subplots': {'RSI': {'rsi': {'color': 'red'}}}}, + {'main_plot': {}, 'subplots': {'RSI': {'rsi': {'color': 'red'}}}}), + # indicator 1 / 2 should have prevelance + (['sma', 'ema3'], ['macd'], + {'main_plot': {'sma': {}}, 'subplots': {'RSI': {'rsi': {'color': 'red'}}}}, + {'main_plot': {'sma': {}, 'ema3': {}}, 'subplots': {'Other': {'macd': {}}}} + ), + # indicator 1 - overrides plot_config main_plot + (['sma', 'ema3'], [], + {'main_plot': {'sma': {}}, 'subplots': {'RSI': {'rsi': {'color': 'red'}}}}, + {'main_plot': {'sma': {}, 'ema3': {}}, 'subplots': {'RSI': {'rsi': {'color': 'red'}}}} + ), + # indicator 2 - overrides plot_config subplots + ([], ['macd', 'macd_signal'], + {'main_plot': {'sma': {}}, 'subplots': {'RSI': {'rsi': {'color': 'red'}}}}, + {'main_plot': {'sma': {}}, 'subplots': {'Other': {'macd': {}, 'macd_signal': {}}}} + ), +]) +def test_create_plotconfig(ind1, ind2, plot_conf, exp): + + res = create_plotconfig(ind1, ind2, plot_conf) + assert 'main_plot' in res + assert 'subplots' in res + assert isinstance(res['main_plot'], dict) + assert isinstance(res['subplots'], dict) + + assert res == exp diff --git a/tests/test_wallets.py b/tests/test_wallets.py index 3177edc05..884470014 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -32,7 +32,7 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets._wallets['GAS'].used == 0.0 assert freqtrade.wallets._wallets['GAS'].total == 0.260739 assert freqtrade.wallets.get_free('BNT') == 1.0 - + assert freqtrade.wallets._last_wallet_refresh > 0 mocker.patch.multiple( 'freqtrade.exchange.Exchange', get_balances=MagicMock(return_value={ @@ -61,6 +61,11 @@ def test_sync_wallet_at_boot(mocker, default_conf): assert freqtrade.wallets.get_free('GAS') == 0.270739 assert freqtrade.wallets.get_used('GAS') == 0.1 assert freqtrade.wallets.get_total('GAS') == 0.260439 + update_mock = mocker.patch('freqtrade.wallets.Wallets._update_live') + freqtrade.wallets.update(False) + assert update_mock.call_count == 0 + freqtrade.wallets.update() + assert update_mock.call_count == 1 def test_sync_wallet_missing_data(mocker, default_conf): diff --git a/tests/test_worker.py b/tests/test_worker.py index 72e215210..839f7cdac 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -5,17 +5,17 @@ from unittest.mock import MagicMock, PropertyMock from freqtrade.data.dataprovider import DataProvider from freqtrade.state import State from freqtrade.worker import Worker -from tests.conftest import get_patched_worker, log_has +from tests.conftest import get_patched_worker, log_has, log_has_re def test_worker_state(mocker, default_conf, markets) -> None: mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets)) worker = get_patched_worker(mocker, default_conf) - assert worker.state is State.RUNNING + assert worker.freqtrade.state is State.RUNNING default_conf.pop('initial_state') worker = Worker(args=None, config=default_conf) - assert worker.state is State.STOPPED + assert worker.freqtrade.state is State.STOPPED def test_worker_running(mocker, default_conf, caplog) -> None: @@ -38,15 +38,13 @@ def test_worker_running(mocker, default_conf, caplog) -> None: def test_worker_stopped(mocker, default_conf, caplog) -> None: mock_throttle = MagicMock() mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) - mock_sleep = mocker.patch('time.sleep', return_value=None) worker = get_patched_worker(mocker, default_conf) - worker.state = State.STOPPED + worker.freqtrade.state = State.STOPPED state = worker._worker(old_state=State.RUNNING) assert state is State.STOPPED assert log_has('Changing state to: STOPPED', caplog) - assert mock_throttle.call_count == 0 - assert mock_sleep.call_count == 1 + assert mock_throttle.call_count == 1 def test_throttle(mocker, default_conf, caplog) -> None: @@ -57,14 +55,14 @@ def test_throttle(mocker, default_conf, caplog) -> None: worker = get_patched_worker(mocker, default_conf) start = time.time() - result = worker._throttle(throttled_func, min_secs=0.1) + result = worker._throttle(throttled_func, throttle_secs=0.1) end = time.time() assert result == 42 assert end - start > 0.1 - assert log_has('Throttling throttled_func for 0.10 seconds', caplog) + assert log_has_re(r"Throttling with 'throttled_func\(\)': sleep for \d\.\d{2} s.*", caplog) - result = worker._throttle(throttled_func, min_secs=-1) + result = worker._throttle(throttled_func, throttle_secs=-1) assert result == 42 @@ -74,8 +72,54 @@ def test_throttle_with_assets(mocker, default_conf) -> None: worker = get_patched_worker(mocker, default_conf) - result = worker._throttle(throttled_func, min_secs=0.1, nb_assets=666) + result = worker._throttle(throttled_func, throttle_secs=0.1, nb_assets=666) assert result == 666 - result = worker._throttle(throttled_func, min_secs=0.1) + result = worker._throttle(throttled_func, throttle_secs=0.1) assert result == -1 + + +def test_worker_heartbeat_running(default_conf, mocker, caplog): + message = r"Bot heartbeat\. PID=.*state='RUNNING'" + + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + worker = get_patched_worker(mocker, default_conf) + + worker.freqtrade.state = State.RUNNING + worker._worker(old_state=State.STOPPED) + assert log_has_re(message, caplog) + + caplog.clear() + # Message is not shown before interval is up + worker._worker(old_state=State.RUNNING) + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + worker._heartbeat_msg -= 70 + worker._worker(old_state=State.RUNNING) + assert log_has_re(message, caplog) + + +def test_worker_heartbeat_stopped(default_conf, mocker, caplog): + message = r"Bot heartbeat\. PID=.*state='STOPPED'" + + mock_throttle = MagicMock() + mocker.patch('freqtrade.worker.Worker._throttle', mock_throttle) + worker = get_patched_worker(mocker, default_conf) + + worker.freqtrade.state = State.STOPPED + worker._worker(old_state=State.RUNNING) + assert log_has_re(message, caplog) + + caplog.clear() + # Message is not shown before interval is up + worker._worker(old_state=State.STOPPED) + assert not log_has_re(message, caplog) + + caplog.clear() + # Set clock - 70 seconds + worker._heartbeat_msg -= 70 + worker._worker(old_state=State.STOPPED) + assert log_has_re(message, caplog) diff --git a/tests/testdata/XRP_ETH-trades.json.gz b/tests/testdata/XRP_ETH-trades.json.gz index 69b92cac8..dad822005 100644 Binary files a/tests/testdata/XRP_ETH-trades.json.gz and b/tests/testdata/XRP_ETH-trades.json.gz differ diff --git a/tests/testdata/XRP_OLD-trades.json.gz b/tests/testdata/XRP_OLD-trades.json.gz new file mode 100644 index 000000000..69b92cac8 Binary files /dev/null and b/tests/testdata/XRP_OLD-trades.json.gz differ diff --git a/user_data/logs/.gitkeep b/user_data/logs/.gitkeep new file mode 100644 index 000000000..e69de29bb