diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1882e3bdf..41b8475ec 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,11 +1,20 @@ { "name": "freqtrade Develop", - - "dockerComposeFile": [ - "docker-compose.yml" + "build": { + "dockerfile": "Dockerfile", + "context": ".." + }, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [ + 8080 ], + "mounts": [ + "source=freqtrade-bashhistory,target=/home/ftuser/commandhistory,type=volume" + ], + // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "ftuser", - "service": "ft_vscode", + "postCreateCommand": "freqtrade create-userdir --userdir user_data/", "workspaceFolder": "/freqtrade/", @@ -25,20 +34,6 @@ "ms-python.vscode-pylance", "davidanson.vscode-markdownlint", "ms-azuretools.vscode-docker", + "vscode-icons-team.vscode-icons", ], - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Uncomment the next line if you want start specific services in your Docker Compose config. - // "runServices": [], - - // Uncomment the next line if you want to keep your containers running after VS Code shuts down. - // "shutdownAction": "none", - - // Uncomment the next line to run commands after the container is created - for example installing curl. - // "postCreateCommand": "sudo apt-get update && apt-get install -y git", - - // Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "ftuser" } diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml deleted file mode 100644 index 20ec247d1..000000000 --- a/.devcontainer/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -version: '3' -services: - ft_vscode: - build: - context: .. - dockerfile: ".devcontainer/Dockerfile" - volumes: - # Allow git usage within container - - "${HOME}/.ssh:/home/ftuser/.ssh:ro" - - "${HOME}/.gitconfig:/home/ftuser/.gitconfig:ro" - - ..:/freqtrade:cached - # Persist bash-history - - freqtrade-vscode-server:/home/ftuser/.vscode-server - - freqtrade-bashhistory:/home/ftuser/commandhistory - # Expose API port - ports: - - "127.0.0.1:8080:8080" - command: /bin/sh -c "while sleep 1000; do :; done" - - -volumes: - freqtrade-vscode-server: - freqtrade-bashhistory: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42959c3b5..eb767efb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,13 +79,13 @@ jobs: - name: Backtesting run: | - cp config_bittrex.json.example config.json + cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy - name: Hyperopt run: | - cp config_bittrex.json.example config.json + cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all @@ -172,13 +172,13 @@ jobs: - name: Backtesting run: | - cp config_bittrex.json.example config.json + cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy - name: Hyperopt run: | - cp config_bittrex.json.example config.json + cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all @@ -239,13 +239,13 @@ jobs: - name: Backtesting run: | - cp config_bittrex.json.example config.json + cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy - name: Hyperopt run: | - cp config_bittrex.json.example config.json + cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all @@ -334,6 +334,7 @@ jobs: runs-on: ubuntu-20.04 if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' + steps: - uses: actions/checkout@v2 @@ -411,3 +412,31 @@ jobs: channel: '#notifications' url: ${{ secrets.SLACK_WEBHOOK }} + + deploy_arm: + needs: [ deploy ] + # Only run on 64bit machines + runs-on: [self-hosted, linux, ARM64] + if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' + + steps: + - uses: actions/checkout@v2 + + - name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF##*/})" + id: extract_branch + + - name: Dockerhub login + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + run: | + echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin + + - name: Build and test and push docker images + env: + IMAGE_NAME: freqtradeorg/freqtrade + BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} + run: | + build_helpers/publish_docker_arm64.sh diff --git a/.gitignore b/.gitignore index 4720ff5cb..16df71194 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,8 @@ target/ #exceptions !*.gitkeep +!config_examples/config_binance.example.json +!config_examples/config_bittrex.example.json +!config_examples/config_ftx.example.json +!config_examples/config_full.example.json +!config_examples/config_kraken.example.json diff --git a/.travis.yml b/.travis.yml index 4535c44cb..f2a6d508d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,12 +26,12 @@ jobs: # - coveralls || true name: pytest - script: - - cp config_bittrex.json.example config.json + - cp config_examples/config_bittrex.example.json config.json - freqtrade create-userdir --userdir user_data - freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy name: backtest - script: - - cp config_bittrex.json.example config.json + - cp config_examples/config_bittrex.example.json config.json - freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily name: hyperopt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 040cf3e98..c4ccc1b9f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Few pointers for contributions: - New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR. - PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished). -If you are unsure, discuss the feature on our [discord server](https://discord.gg/p7nuUNVfP7), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. +If you are unsure, discuss the feature on our [discord server](https://discord.gg/p7nuUNVfP7) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a Pull Request. ## Getting started diff --git a/README.md b/README.md index 4082995f0..8cba78136 100644 --- a/README.md +++ b/README.md @@ -142,13 +142,9 @@ The project is currently setup in two main branches: ## Support -### Help / Discord / Slack +### Help / Discord -For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. - -Please check out our [discord server](https://discord.gg/p7nuUNVfP7). - -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). +For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join the Freqtrade [discord server](https://discord.gg/p7nuUNVfP7). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) @@ -179,7 +175,7 @@ to understand the requirements before sending your pull-requests. Coding is not a necessity to contribute - maybe start with improving our documentation? Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. -**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/p7nuUNVfP7) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/p7nuUNVfP7) (please use the #dev channel for this). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. **Important:** Always create your PR against the `develop` branch, not `stable`. diff --git a/build_helpers/TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl deleted file mode 100644 index b4eee9c47..000000000 Binary files a/build_helpers/TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.20-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.20-cp38-cp38-win_amd64.whl deleted file mode 100644 index de7393fde..000000000 Binary files a/build_helpers/TA_Lib-0.4.20-cp38-cp38-win_amd64.whl and /dev/null differ diff --git a/build_helpers/TA_Lib-0.4.21-cp37-cp37m-win_amd64.whl b/build_helpers/TA_Lib-0.4.21-cp37-cp37m-win_amd64.whl new file mode 100644 index 000000000..bccfd090f Binary files /dev/null and b/build_helpers/TA_Lib-0.4.21-cp37-cp37m-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.21-cp38-cp38-win_amd64.whl b/build_helpers/TA_Lib-0.4.21-cp38-cp38-win_amd64.whl new file mode 100644 index 000000000..67b41bf99 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.21-cp38-cp38-win_amd64.whl differ diff --git a/build_helpers/TA_Lib-0.4.21-cp39-cp39-win_amd64.whl b/build_helpers/TA_Lib-0.4.21-cp39-cp39-win_amd64.whl new file mode 100644 index 000000000..da9d74558 Binary files /dev/null and b/build_helpers/TA_Lib-0.4.21-cp39-cp39-win_amd64.whl differ diff --git a/build_helpers/install_windows.ps1 b/build_helpers/install_windows.ps1 index d2a27dd26..ec38ea212 100644 --- a/build_helpers/install_windows.ps1 +++ b/build_helpers/install_windows.ps1 @@ -6,10 +6,13 @@ python -m pip install --upgrade pip $pyv = python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" if ($pyv -eq '3.7') { - pip install build_helpers\TA_Lib-0.4.20-cp37-cp37m-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.21-cp37-cp37m-win_amd64.whl } if ($pyv -eq '3.8') { - pip install build_helpers\TA_Lib-0.4.20-cp38-cp38-win_amd64.whl + pip install build_helpers\TA_Lib-0.4.21-cp38-cp38-win_amd64.whl +} +if ($pyv -eq '3.9') { + pip install build_helpers\TA_Lib-0.4.21-cp39-cp39-win_amd64.whl } pip install -r requirements-dev.txt diff --git a/build_helpers/publish_docker_arm64.sh b/build_helpers/publish_docker_arm64.sh new file mode 100755 index 000000000..756d5e41d --- /dev/null +++ b/build_helpers/publish_docker_arm64.sh @@ -0,0 +1,80 @@ +#!/bin/sh + +# Use BuildKit, otherwise building on ARM fails +export DOCKER_BUILDKIT=1 + +# Replace / with _ to create a valid tag +TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") +TAG_PLOT=${TAG}_plot +TAG_PI="${TAG}_pi" + +TAG_ARM=${TAG}_arm +TAG_PLOT_ARM=${TAG_PLOT}_arm +CACHE_IMAGE=freqtradeorg/freqtrade_cache + +echo "Running for ${TAG}" + +# Add commit and commit_message to docker container +echo "${GITHUB_SHA}" > freqtrade_commit + +if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then + echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" + # Build regular image + docker build -t freqtrade:${TAG_ARM} . + +else + echo "event ${GITHUB_EVENT_NAME}: building with cache" + # Build regular image + docker pull ${IMAGE_NAME}:${TAG_ARM} + docker build --cache-from ${IMAGE_NAME}:${TAG_ARM} -t freqtrade:${TAG_ARM} . + +fi + +if [ $? -ne 0 ]; then + echo "failed building multiarch images" + return 1 +fi +# Tag image for upload and next build step +docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM + +docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot . + +docker tag freqtrade:$TAG_PLOT_ARM ${CACHE_IMAGE}:$TAG_PLOT_ARM + +# Run backtest +docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy + +if [ $? -ne 0 ]; then + echo "failed running backtest" + return 1 +fi + +docker images + +# docker push ${IMAGE_NAME} +docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM +docker push ${CACHE_IMAGE}:$TAG_ARM + +# Create multi-arch image +# Make sure that all images contained here are pushed to github first. +# Otherwise installation might fail. +echo "create manifests" + +docker manifest create --amend ${IMAGE_NAME}:${TAG} ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG} +docker manifest push -p ${IMAGE_NAME}:${TAG} + +docker manifest create --amend ${IMAGE_NAME}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} ${CACHE_IMAGE}:${TAG_PLOT} +docker manifest push -p ${IMAGE_NAME}:${TAG_PLOT} + +Tag as latest for develop builds +if [ "${TAG}" = "develop" ]; then + docker tag ${IMAGE_NAME}:develop ${IMAGE_NAME}:latest + docker push ${IMAGE_NAME}:latest +fi + +docker images + +if [ $? -ne 0 ]; then + echo "failed building image" + return 1 +fi diff --git a/build_helpers/publish_docker_multi.sh b/build_helpers/publish_docker_multi.sh index a6b06ce7d..4961cb9a7 100755 --- a/build_helpers/publish_docker_multi.sh +++ b/build_helpers/publish_docker_multi.sh @@ -9,7 +9,8 @@ TAG_PI="${TAG}_pi" PI_PLATFORM="linux/arm/v7" echo "Running for ${TAG}" -CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache +CACHE_IMAGE=freqtradeorg/freqtrade_cache +CACHE_TAG=${CACHE_IMAGE}:${TAG_PI}_cache # Add commit and commit_message to docker container echo "${GITHUB_SHA}" > freqtrade_commit @@ -45,14 +46,14 @@ if [ $? -ne 0 ]; then return 1 fi # Tag image for upload and next build step -docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG +docker tag freqtrade:$TAG ${CACHE_IMAGE}:$TAG docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot . -docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT +docker tag freqtrade:$TAG_PLOT ${CACHE_IMAGE}:$TAG_PLOT # Run backtest -docker run --rm -v $(pwd)/config_bittrex.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy +docker run --rm -v $(pwd)/config_examples/config_bittrex.example.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy if [ $? -ne 0 ]; then echo "failed running backtest" @@ -61,22 +62,9 @@ fi docker images -docker push ${IMAGE_NAME} -docker push ${IMAGE_NAME}:$TAG_PLOT -docker push ${IMAGE_NAME}:$TAG - -# Create multiarch image -# Make sure that all images contained here are pushed to github first. -# Otherwise installation might fail. - -docker manifest create freqtradeorg/freqtrade:${TAG} ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} -docker manifest push freqtradeorg/freqtrade:${TAG} - -# Tag as latest for develop builds -if [ "${TAG}" = "develop" ]; then - docker manifest create freqtradeorg/freqtrade:latest ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} - docker manifest push freqtradeorg/freqtrade:latest -fi +docker push ${CACHE_IMAGE} +docker push ${CACHE_IMAGE}:$TAG_PLOT +docker push ${CACHE_IMAGE}:$TAG docker images diff --git a/config_binance.json.example b/config_examples/config_binance.example.json similarity index 100% rename from config_binance.json.example rename to config_examples/config_binance.example.json diff --git a/config_bittrex.json.example b/config_examples/config_bittrex.example.json similarity index 100% rename from config_bittrex.json.example rename to config_examples/config_bittrex.example.json diff --git a/config_ftx.json.example b/config_examples/config_ftx.example.json similarity index 100% rename from config_ftx.json.example rename to config_examples/config_ftx.example.json diff --git a/config_full.json.example b/config_examples/config_full.example.json similarity index 100% rename from config_full.json.example rename to config_examples/config_full.example.json diff --git a/config_kraken.json.example b/config_examples/config_kraken.example.json similarity index 100% rename from config_kraken.json.example rename to config_examples/config_kraken.example.json diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 35fd3de4a..5e71df67c 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -32,6 +32,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss): def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, config: Dict, processed: Dict[str, DataFrame], + backtest_stats: Dict[str, Any], *args, **kwargs) -> float: """ Objective function, returns smaller number for better results @@ -53,7 +54,7 @@ class SuperDuperHyperOptLoss(IHyperOptLoss): Currently, the arguments are: -* `results`: DataFrame containing the result +* `results`: DataFrame containing the resulting trades. The following columns are available in results (corresponds to the output-file of backtesting when used with `--export trades`): `pair, profit_ratio, profit_abs, open_date, open_rate, fee_open, close_date, close_rate, fee_close, amount, trade_duration, is_open, sell_reason, stake_amount, min_rate, max_rate, stop_loss_ratio, stop_loss_abs` * `trade_count`: Amount of trades (identical to `len(results)`) @@ -61,6 +62,7 @@ Currently, the arguments are: * `min_date`: End date of the timerange used * `config`: Config object used (Note: Not all strategy-related parameters will be updated here if they are part of a hyperopt space). * `processed`: Dict of Dataframes with the pair as keys containing the data used for backtesting. +* `backtest_stats`: Backtesting statistics using the same format as the backtesting file "strategy" substructure. Available fields can be seen in `generate_strategy_stats()` in `optimize_reports.py`. This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. diff --git a/docs/backtesting.md b/docs/backtesting.md index 4899b1dad..89980c670 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -302,7 +302,6 @@ A backtesting result will look like that: | Days win/draw/lose | 12 / 82 / 25 | | Avg. Duration Winners | 4:23:00 | | Avg. Duration Loser | 6:55:00 | -| Zero Duration Trades | 4.6% (20) | | Rejected Buy signals | 3089 | | | | | Min balance | 0.00945123 BTC | @@ -390,7 +389,6 @@ It contains some useful key metrics about performance of your strategy on backte | Days win/draw/lose | 12 / 82 / 25 | | Avg. Duration Winners | 4:23:00 | | Avg. Duration Loser | 6:55:00 | -| Zero Duration Trades | 4.6% (20) | | Rejected Buy signals | 3089 | | | | | Min balance | 0.00945123 BTC | @@ -420,7 +418,6 @@ It contains some useful key metrics about performance of your strategy on backte - `Best day` / `Worst day`: Best and worst day based on daily profit. - `Days win/draw/lose`: Winning / Losing days (draws are usually days without closed trade). - `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades. -- `Zero Duration Trades`: A number of trades that completed within same candle as they opened and had `trailing_stop_loss` sell reason. A significant amount of such trades may indicate that strategy is exploiting trailing stoploss behavior in backtesting and produces unrealistic results. - `Rejected Buy signals`: Buy signals that could not be acted upon due to max_open_trades being reached. - `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period. - `Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced). diff --git a/docs/configuration.md b/docs/configuration.md index 5c6236e58..e1c7e77d5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -52,6 +52,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `stake_currency` | **Required.** Crypto-currency used for trading.
**Datatype:** String | `stake_amount` | **Required.** Amount of crypto-currency your bot will use for each trade. Set it to `"unlimited"` to allow the bot to use all available balance. [More information below](#configuring-amount-per-trade).
**Datatype:** Positive float or `"unlimited"`. | `tradable_balance_ratio` | Ratio of the total account balance the bot is allowed to trade. [More information below](#configuring-amount-per-trade).
*Defaults to `0.99` 99%).*
**Datatype:** Positive float between `0.1` and `1.0`. +| `available_capital` | Available starting capital for the bot. Useful when running multiple bots on the same exchange account.[More information below](#configuring-amount-per-trade).
**Datatype:** Positive float. | `amend_last_stake_amount` | Use reduced last stake amount if necessary. [More information below](#configuring-amount-per-trade).
*Defaults to `false`.*
**Datatype:** Boolean | `last_stake_amount_min_ratio` | Defines minimum stake amount that has to be left and executed. Applies only to the last stake amount when it's amended to a reduced value (i.e. if `amend_last_stake_amount` is set to `true`). [More information below](#configuring-amount-per-trade).
*Defaults to `0.5`.*
**Datatype:** Float (as ratio) | `amount_reserve_percent` | Reserve some amount in min pair stake amount. The bot will reserve `amount_reserve_percent` + stoploss value when calculating min pair stake amount in order to avoid possible trade refusals.
*Defaults to `0.05` (5%).*
**Datatype:** Positive Float as ratio. @@ -164,7 +165,7 @@ Values set in the configuration file always overwrite values set in the strategy ### Configuring amount per trade -There are several methods to configure how much of the stake currency the bot will use to enter a trade. All methods respect the [available balance configuration](#available-balance) as explained below. +There are several methods to configure how much of the stake currency the bot will use to enter a trade. All methods respect the [available balance configuration](#tradable-balance) as explained below. #### Minimum trade stake @@ -183,7 +184,7 @@ To limit this calculation in case of large stoploss values, the calculated minim !!! Warning Since the limits on exchanges are usually stable and are not updated often, some pairs can show pretty high minimum limits, simply because the price increased a lot since the last limit adjustment by the exchange. -#### Available balance +#### Tradable balance By default, the bot assumes that the `complete amount - 1%` is at it's disposal, and when using [dynamic stake amount](#dynamic-stake-amount), it will split the complete balance into `max_open_trades` buckets per trade. Freqtrade will reserve 1% for eventual fees when entering a trade and will therefore not touch that by default. @@ -192,9 +193,25 @@ You can configure the "untouched" amount by using the `tradable_balance_ratio` s For example, if you have 10 ETH available in your wallet on the exchange and `tradable_balance_ratio=0.5` (which is 50%), then the bot will use a maximum amount of 5 ETH for trading and considers this as available balance. The rest of the wallet is untouched by the trades. +!!! Danger + This setting should **not** be used when running multiple bots on the same account. Please look at [Available Capital to the bot](#assign-available-capital) instead. + !!! Warning The `tradable_balance_ratio` setting applies to the current balance (free balance + tied up in trades). Therefore, assuming the starting balance of 1000, a configuration with `tradable_balance_ratio=0.99` will not guarantee that 10 currency units will always remain available on the exchange. For example, the free amount may reduce to 5 units if the total balance is reduced to 500 (either by a losing streak, or by withdrawing balance). +#### Assign available Capital + +To fully utilize compounding profits when using multiple bots on the same exchange account, you'll want to limit each bot to a certain starting balance. +This can be accomplished by setting `available_capital` to the desired starting balance. + +Assuming your account has 10.000 USDT and you want to run 2 different strategies on this exchange. +You'd set `available_capital=5000` - granting each bot an initial capital of 5000 USDT. +The bot will then split this starting balance equally into `max_open_trades` buckets. +Profitable trades will result in increased stake-sizes for this bot - without affecting stake-sizes of the other bot. + +!!! Warning "Incompatible with `tradable_balance_ratio`" + Setting this option will replace any configuration of `tradable_balance_ratio`. + #### Amend last stake amount Assuming we have the tradable balance of 1000 USDT, `stake_amount=400`, and `max_open_trades=3`. @@ -556,7 +573,7 @@ You should also make sure to read the [Exchanges](exchanges.md) section of the d To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration. -An example for this can be found in `config_full.json.example` +An example for this can be found in `config_examples/config_full.example.json` ``` json "ccxt_async_config": { diff --git a/docs/developer.md b/docs/developer.md index d0731a233..dd56a367c 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/p7nuUNVfP7) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/p7nuUNVfP7) where you can ask questions. ## Documentation diff --git a/docs/faq.md b/docs/faq.md index d015ae50e..b8a3a44d8 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -172,7 +172,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD ### Why does it take a long time to run hyperopt? -* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/p7nuUNVfP7). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. +* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [discord community](https://discord.gg/p7nuUNVfP7). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers: diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 5ce0d3813..4fba925d0 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -51,7 +51,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]] [--print-all] [--no-color] [--print-json] [-j JOBS] [--random-state INT] [--min-trades INT] - [--hyperopt-loss NAME] + [--hyperopt-loss NAME] [--disable-param-export] optional arguments: -h, --help show this help message and exit @@ -118,6 +118,8 @@ optional arguments: ShortTradeDurHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, SortinoHyperOptLoss, SortinoHyperOptLossDaily + --disable-param-export + Disable automatic hyperopt parameter export. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -512,7 +514,13 @@ You should understand this result like: * You should not use ADX because `'buy_adx_enabled': False`. * You should **consider** using the RSI indicator (`'buy_rsi_enabled': True`) and the best value is `29.0` (`'buy_rsi': 29.0`) -Your strategy class can immediately take advantage of these results. Simply copy hyperopt results block and paste them at class level, replacing old parameters (if any). New parameters will automatically be loaded next time strategy is executed. +### Automatic parameter application to the strategy + +When using Hyperoptable parameters, the result of your hyperopt-run will be written to a json file next to your strategy (so for `MyAwesomeStrategy.py`, the file would be `MyAwesomeStrategy.json`). +This file is also updated when using the `hyperopt-show` sub-command, unless `--disable-param-export` is provided to either of the 2 commands. + + +Your strategy class can also contain these results explicitly. Simply copy hyperopt results block and paste them at class level, replacing old parameters (if any). New parameters will automatically be loaded next time strategy is executed. Transferring your whole hyperopt result to your strategy would then look like: @@ -528,6 +536,10 @@ class MyAwesomeStrategy(IStrategy): } ``` +!!! Note + Values in the configuration file will overwrite Parameter-file level parameters - and both will overwrite parameters within the strategy. + The prevalence is therefore: config > parameter file > strategy + ### Understand Hyperopt ROI results If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table: diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index f19c5a181..8727cc3fc 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -23,6 +23,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) * [`AgeFilter`](#agefilter) +* [`OffsetFilter`](#offsetfilter) * [`PerformanceFilter`](#performancefilter) * [`PrecisionFilter`](#precisionfilter) * [`PriceFilter`](#pricefilter) @@ -63,17 +64,56 @@ The `refresh_period` setting allows to define the period (in seconds), at which The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists. Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data. -`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library: +`VolumePairList` is per default based on the ticker data from exchange, as reported by the ccxt library: * The `quoteVolume` is the amount of quote (stake) currency traded (bought or sold) in last 24 hours. ```json -"pairlists": [{ +"pairlists": [ + { "method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume", "refresh_period": 1800 -}], + } +], +``` + +`VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles. + +For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days: + +```json +"pairlists": [ + { + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "refresh_period": 86400, + "lookback_days": 7 + } +], +``` + +!!! Warning "Range look back and refresh period" + When used in conjunction with `lookback_days` and `lookback_timeframe` the `refresh_period` can not be smaller than the candle size in seconds. As this will result in unnecessary requests to the exchanges API. + +!!! Warning "Performance implications when using lookback range" + If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation. + +More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles: + +```json +"pairlists": [ + { + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "refresh_period": 3600, + "lookback_timeframe": "1h", + "lookback_period": 72 + } +], ``` !!! Note @@ -81,13 +121,39 @@ Filtering instances (not the first position in the list) will not apply any cach #### AgeFilter -Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`). +Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`) or more than `max_days_listed` days (defaults `None` mean infinity). When pairs are first listed on an exchange they can suffer huge price drops and volatility in the first few days while the pair goes through its price-discovery period. Bots can often be caught out buying before the pair has finished dropping in price. -This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days. +This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days and listed before `max_days_listed`. + +#### OffsetFilter + +Offsets an incoming pairlist by a given `offset` value. + +As an example it can be used in conjunction with `VolumeFilter` to remove the top X volume pairs. Or to split +a larger pairlist on two bot instances. + +Example to remove the first 10 pairs from the pairlist: + +```json +"pairlists": [ + { + "method": "OffsetFilter", + "offset": 10 + } +], +``` + +!!! Warning + When `OffsetFilter` is used to split a larger pairlist among multiple bots in combination with `VolumeFilter` + it can not be guaranteed that pairs won't overlap due to slightly different refresh intervals for the + `VolumeFilter`. + +!!! Note + An offset larger then the total length of the incoming pairlist will result in an empty pairlist. #### PerformanceFilter diff --git a/docs/includes/protections.md b/docs/includes/protections.md index 3ea2dde61..5dcc83738 100644 --- a/docs/includes/protections.md +++ b/docs/includes/protections.md @@ -1,7 +1,7 @@ ## Protections !!! Warning "Beta feature" - This feature is still in it's testing phase. Should you notice something you think is wrong please let us know via Discord, Slack or via Github Issue. + This feature is still in it's testing phase. Should you notice something you think is wrong please let us know via Discord or via Github Issue. Protections will protect your strategy from unexpected events and market conditions by temporarily stop trading for either one pair, or for all pairs. All protection end times are rounded up to the next candle to avoid sudden, unexpected intra-candle buys. diff --git a/docs/index.md b/docs/index.md index 8ecb085de..8077cd303 100644 --- a/docs/index.md +++ b/docs/index.md @@ -73,13 +73,9 @@ Alternatively ## Support -### Help / Discord / Slack +### Help / Discord -For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. - -Please check out our [discord server](https://discord.gg/p7nuUNVfP7). - -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). +For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join the Freqtrade [discord server](https://discord.gg/p7nuUNVfP7). ## Ready to try? diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index d11e5ea4e..32a1df58e 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ -mkdocs==1.2.1 -mkdocs-material==7.1.9 +mkdocs==1.2.2 +mkdocs-material==7.1.11 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index dd38d2cbc..0b52af17b 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -55,7 +55,7 @@ class AwesomeStrategy(IStrategy): dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe) # Obtain last available candle. Do not use current_time to look up latest candle, because - # current_time points to curret incomplete candle whose data is not available. + # current_time points to current incomplete candle whose data is not available. last_candle = dataframe.iloc[-1].squeeze() # <...> @@ -83,7 +83,7 @@ It is possible to define custom sell signals, indicating that specified position For example you could implement a 1:2 risk-reward ROI with `custom_sell()`. -Using custom_sell() signals in place of stoplosses though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange. +Using custom_sell() signals in place of stoploss though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange. !!! Note Returning a `string` or `True` from this method is equal to setting sell signal on a candle at specified time. This method is not called when sell signal is set already, or if sell signals are disabled (`use_sell_signal=False` or `sell_profit_only=True` while profit is below `sell_profit_offset`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters. @@ -269,7 +269,7 @@ class AwesomeStrategy(IStrategy): current_rate: float, current_profit: float, **kwargs) -> float: if current_profit < 0.04: - return -1 # return a value bigger than the inital stoploss to keep using the inital stoploss + return -1 # return a value bigger than the initial stoploss to keep using the initial stoploss # After reaching the desired offset, allow the stoploss to trail by half the profit desired_stoploss = current_profit / 2 @@ -547,6 +547,39 @@ class AwesomeStrategy(IStrategy): ``` +### Stake size management + +It is possible to manage your risk by reducing or increasing stake amount when placing a new trade. + +```python +class AwesomeStrategy(IStrategy): + def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float, + proposed_stake: float, min_stake: float, max_stake: float, + **kwargs) -> float: + + dataframe, _ = self.dp.get_analyzed_dataframe(pair=pair, timeframe=self.timeframe) + current_candle = dataframe.iloc[-1].squeeze() + + if current_candle['fastk_rsi_1h'] > current_candle['fastd_rsi_1h']: + if self.config['stake_amount'] == 'unlimited': + # Use entire available wallet during favorable conditions when in compounding mode. + return max_stake + else: + # Compound profits during favorable conditions instead of using a static stake. + return self.wallets.get_total_stake_amount() / self.config['max_open_trades'] + + # Use default stake amount. + return proposed_stake +``` + +Freqtrade will fall back to the `proposed_stake` value should your code raise an exception. The exception itself will be logged. + +!!! Tip + You do not _have_ to ensure that `min_stake <= returned_value <= max_stake`. Trades will succeed as the returned value will be clamped to supported range and this acton will be logged. + +!!! Tip + Returning `0` or `None` will prevent trades from being placed. + --- ## Derived strategies diff --git a/docs/strategy_analysis_example.md b/docs/strategy_analysis_example.md index 4c938500c..27192aa2f 100644 --- a/docs/strategy_analysis_example.md +++ b/docs/strategy_analysis_example.md @@ -130,6 +130,44 @@ trades = load_backtest_data(backtest_dir) trades.groupby("pair")["sell_reason"].value_counts() ``` +## Plotting daily profit / equity line + + +```python +# Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day) + +from freqtrade.configuration import Configuration +from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats +import plotly.express as px +import pandas as pd + +# strategy = 'SampleStrategy' +# config = Configuration.from_files(["user_data/config.json"]) +# backtest_dir = config["user_data_dir"] / "backtest_results" + +stats = load_backtest_stats(backtest_dir) +strategy_stats = stats['strategy'][strategy] + +dates = [] +profits = [] +for date_profit in strategy_stats['daily_profit']: + dates.append(date_profit[0]) + profits.append(date_profit[1]) + +equity = 0 +equity_daily = [] +for daily_profit in profits: + equity_daily.append(equity) + equity += float(daily_profit) + + +df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily}) + +fig = px.line(df, x="dates", y="equity_daily") +fig.show() + +``` + ### Load live trading results into a pandas dataframe In case you did already some trading and want to analyze your performance diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index f5d9744b4..b020b00db 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -245,10 +245,10 @@ current max Return a summary of your profit/loss and performance. > **ROI:** Close trades -> ∙ `0.00485701 BTC (258.45%)` +> ∙ `0.00485701 BTC (2.2%) (15.2 Σ%)` > ∙ `62.968 USD` > **ROI:** All trades -> ∙ `0.00255280 BTC (143.43%)` +> ∙ `0.00255280 BTC (1.5%) (6.43 Σ%)` > ∙ `33.095 EUR` > > **Total Trade Count:** `138` @@ -257,6 +257,10 @@ Return a summary of your profit/loss and performance. > **Avg. Duration:** `2:33:45` > **Best Performing:** `PAY/BTC: 50.23%` +The relative profit of `1.2%` is the average profit per trade. +The relative profit of `15.2 Σ%` is be based on the starting capital - so in this case, the starting capital was `0.00485701 * 1.152 = 0.00738 BTC`. +Starting capital is either taken from the `available_capital` setting, or calculated by using current wallet size - profits. + ### /forcesell > **BITTREX:** Selling BTC/LTC with limit `0.01650000 (profit: ~-4.07%, -0.00008168)` diff --git a/docs/utils.md b/docs/utils.md index 8ef12e1c9..789462de4 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -614,6 +614,48 @@ Show whitelist when using a [dynamic pairlist](plugins.md#pairlists). freqtrade test-pairlist --config config.json --quote USDT BTC ``` +## Webserver mode + +!!! Warning "Experimental" + Webserver mode is an experimental mode to increase backesting and strategy development productivity. + There may still be bugs - so if you happen to stumble across these, please report them as github issues, thanks. + +Run freqtrade in webserver mode. +Freqtrade will start the webserver and allow FreqUI to start and control backtesting processes. +This has the advantage that data will not be reloaded between backtesting runs (as long as timeframe and timerange remain identical). +FreqUI will also show the backtesting results. + +``` +usage: freqtrade webserver [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] + [--userdir PATH] [-s NAME] [--strategy-path PATH] + +optional arguments: + -h, --help show this help message and exit + +Common arguments: + -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). + --logfile FILE Log to the file specified. Special values are: + 'syslog', 'journald'. See the documentation for more + details. + -V, --version show program's version number and exit + -c PATH, --config PATH + Specify configuration file (default: + `userdir/config.json` or `config.json` whichever + exists). Multiple --config options may be used. Can be + set to `-` to read config from stdin. + -d PATH, --datadir PATH + Path to directory with historical backtesting data. + --userdir PATH, --user-data-dir PATH + Path to userdata directory. + +Strategy arguments: + -s NAME, --strategy NAME + Specify strategy class name which will be used by the + bot. + --strategy-path PATH Specify additional strategy lookup path. + +``` + ## List Hyperopt results You can list the hyperoptimization epochs the Hyperopt module evaluated previously with the `hyperopt-list` sub-command. @@ -702,7 +744,8 @@ You can show the details of any hyperoptimization epoch previously evaluated by usage: freqtrade hyperopt-show [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--best] [--profitable] [-n INT] [--print-json] - [--hyperopt-filename PATH] [--no-header] + [--hyperopt-filename FILENAME] [--no-header] + [--disable-param-export] optional arguments: -h, --help show this help message and exit @@ -714,6 +757,8 @@ optional arguments: Hyperopt result filename.Example: `--hyperopt- filename=hyperopt_results_2020-09-27_16-20-48.pickle` --no-header Do not print epoch details header. + --disable-param-export + Disable automatic hyperopt parameter export. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/windows_installation.md b/docs/windows_installation.md index edc0a1404..2db0ae913 100644 --- a/docs/windows_installation.md +++ b/docs/windows_installation.md @@ -23,7 +23,7 @@ git clone https://github.com/freqtrade/freqtrade.git Install ta-lib according to the [ta-lib documentation](https://github.com/mrjbq7/ta-lib#windows). -As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which needs to be downloaded and installed using `pip install TA_Lib‑0.4.20‑cp38‑cp38‑win_amd64.whl` (make sure to use the version matching your python version). +As compiling from source on windows has heavy dependencies (requires a partial visual studio installation), there is also a repository of unofficial pre-compiled windows Wheels [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#ta-lib), which need to be downloaded and installed using `pip install TA_Lib-0.4.21-cp38-cp38-win_amd64.whl` (make sure to use the version matching your python version). Freqtrade provides these dependencies for the latest 2 Python versions (3.7 and 3.8) and for 64bit Windows. Other versions must be downloaded from the above link. diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 784b99bed..04e46ee23 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -20,3 +20,4 @@ from freqtrade.commands.optimize_commands import start_backtesting, start_edge, from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit from freqtrade.commands.trade_commands import start_trading +from freqtrade.commands.webserver_commands import start_webserver diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 7f4f7edd6..1143db394 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -16,6 +16,8 @@ ARGS_STRATEGY = ["strategy", "strategy_path"] ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"] +ARGS_WEBSERVER: List[str] = [] + ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", "max_open_trades", "stake_amount", "fee", "pairs"] @@ -29,7 +31,7 @@ ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", "epochs", "spaces", "print_all", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", - "hyperopt_loss"] + "hyperopt_loss", "disableparamexport"] ARGS_EDGE = ARGS_COMMON_OPTIMIZE + ["stoploss_range"] @@ -85,7 +87,8 @@ ARGS_HYPEROPT_LIST = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperoptexportfilename", "export_csv"] ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperopt_show_index", - "print_json", "hyperoptexportfilename", "hyperopt_show_no_header"] + "print_json", "hyperoptexportfilename", "hyperopt_show_no_header", + "disableparamexport"] NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", @@ -175,7 +178,8 @@ class Arguments: start_list_markets, start_list_strategies, start_list_timeframes, start_new_config, start_new_hyperopt, start_new_strategy, start_plot_dataframe, start_plot_profit, - start_show_trades, start_test_pairlist, start_trading) + start_show_trades, start_test_pairlist, start_trading, + start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -383,3 +387,9 @@ class Arguments: ) plot_profit_cmd.set_defaults(func=start_plot_profit) self._build_args(optionlist=ARGS_PLOT_PROFIT, parser=plot_profit_cmd) + + # Add webserver subcommand + webserver_cmd = subparsers.add_parser('webserver', help='Webserver module.', + parents=[_common_parser]) + webserver_cmd.set_defaults(func=start_webserver) + self._build_args(optionlist=ARGS_WEBSERVER, parser=webserver_cmd) diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index b226415e7..f56a2bf18 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -178,6 +178,11 @@ AVAILABLE_CLI_OPTIONS = { 'Example: `--export-filename=user_data/backtest_results/backtest_today.json`', metavar='PATH', ), + "disableparamexport": Arg( + '--disable-param-export', + help="Disable automatic hyperopt parameter export.", + action='store_true', + ), "fee": Arg( '--fee', help='Specify fee ratio. Will be applied twice (on trade entry and exit).', diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 3877e0801..141e85f14 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -48,7 +48,8 @@ def start_download_data(args: Dict[str, Any]) -> None: # Init exchange exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) # Manual validations of relevant settings - exchange.validate_pairs(config['pairs']) + if not config['exchange'].get('skip_pair_validation', False): + exchange.validate_pairs(config['pairs']) expanded_pairs = expand_pairlist(config['pairs'], list(exchange.markets)) logger.info(f"About to download pairs: {expanded_pairs}, " diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 19337b407..5a2727795 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -129,9 +129,12 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: metrics = val['results_metrics'] if 'strategy_name' in metrics: - show_backtest_result(metrics['strategy_name'], metrics, + strategy_name = metrics['strategy_name'] + show_backtest_result(strategy_name, metrics, metrics['stake_currency']) + HyperoptTools.try_export_params(config, strategy_name, val) + HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, header_str="Epoch details") diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index cd26aa60e..410b9b72b 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -14,7 +14,7 @@ from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active, validate_exchanges -from freqtrade.misc import plural +from freqtrade.misc import parse_db_uri_for_logging, plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -225,7 +225,7 @@ def start_show_trades(args: Dict[str, Any]) -> None: if 'db_url' not in config: raise OperationalException("--db-url is required for this command.") - logger.info(f'Using DB: "{config["db_url"]}"') + logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"') init_db(config['db_url'], clean_open_orders=False) tfilter = [] diff --git a/freqtrade/commands/webserver_commands.py b/freqtrade/commands/webserver_commands.py new file mode 100644 index 000000000..9a5975227 --- /dev/null +++ b/freqtrade/commands/webserver_commands.py @@ -0,0 +1,15 @@ +from typing import Any, Dict + +from freqtrade.enums import RunMode + + +def start_webserver(args: Dict[str, Any]) -> None: + """ + Main entry point for webserver mode + """ + from freqtrade.configuration import Configuration + from freqtrade.rpc.api_server import ApiServer + + # Initialize configuration + config = Configuration(args, RunMode.WEBSERVER).get_config() + ApiServer(config, standalone=True) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index d2cc68c44..bd30adcae 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -15,7 +15,7 @@ from freqtrade.configuration.load_config import load_config_file, load_file from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging -from freqtrade.misc import deep_merge_dicts +from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging logger = logging.getLogger(__name__) @@ -71,7 +71,7 @@ class Configuration: # Merge config options, overwriting old values config = deep_merge_dicts(load_config_file(path), config) - + config['config_files'] = files # Normalize config if 'internals' not in config: config['internals'] = {} @@ -144,7 +144,7 @@ class Configuration: config['db_url'] = constants.DEFAULT_DB_PROD_URL logger.info('Dry run is disabled') - logger.info(f'Using DB: "{config["db_url"]}"') + logger.info(f'Using DB: "{parse_db_uri_for_logging(config["db_url"])}"') def _process_common_options(self, config: Dict[str, Any]) -> None: @@ -260,6 +260,8 @@ class Configuration: self._args_to_config(config, argname='export', logstring='Parameter --export detected: {} ...') + self._args_to_config(config, argname='disableparamexport', + logstring='Parameter --disableparamexport detected: {} ...') # Edge section: if 'stoploss_range' in self.args and self.args["stoploss_range"]: txt_range = eval(self.args["stoploss_range"]) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 63cf3e870..2f93ace1c 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -26,9 +26,9 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', - 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', - 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', - 'SpreadFilter', 'VolatilityFilter'] + 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', + 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', + 'ShuffleFilter', 'SpreadFilter', 'VolatilityFilter'] AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 @@ -40,6 +40,7 @@ DEFAULT_DATAFRAME_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume'] DEFAULT_TRADES_COLUMNS = ['timestamp', 'id', 'type', 'side', 'price', 'amount', 'cost'] LAST_BT_RESULT_FN = '.last_result.json' +FTHYPT_FILEVERSION = 'fthypt_fileversion' USERPATH_HYPEROPTS = 'hyperopts' USERPATH_STRATEGIES = 'strategies' @@ -112,6 +113,10 @@ CONF_SCHEMA = { 'maximum': 1, 'default': 0.99 }, + 'available_capital': { + 'type': 'number', + 'minimum': 0, + }, 'amend_last_stake_amount': {'type': 'boolean', 'default': False}, 'last_stake_amount_min_ratio': { 'type': 'number', 'minimum': 0.0, 'maximum': 1.0, 'default': 0.5 @@ -312,6 +317,7 @@ CONF_SCHEMA = { }, 'db_url': {'type': 'string'}, 'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'}, + 'disableparamexport': {'type': 'boolean'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'forcebuy_enable': {'type': 'boolean'}, 'disable_dataframe_checks': {'type': 'boolean'}, diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index eecb63d07..1459dfd78 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -194,8 +194,8 @@ def _download_pair_history(datadir: Path, new_data = exchange.get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms if since_ms else - int(arrow.utcnow().shift( - days=-new_pairs_days).float_timestamp) * 1000 + arrow.utcnow().shift( + days=-new_pairs_days).int_timestamp * 1000 ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, @@ -272,7 +272,7 @@ def _download_trades_history(exchange: Exchange, if timerange.stoptype == 'date': until = timerange.stopts * 1000 else: - since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000 + since = arrow.utcnow().shift(days=-new_pairs_days).int_timestamp * 1000 trades = data_handler.trades_load(pair) diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py index bcf68d622..60f984c33 100644 --- a/freqtrade/enums/__init__.py +++ b/freqtrade/enums/__init__.py @@ -1,4 +1,5 @@ # flake8: noqa: F401 +from freqtrade.enums.backteststate import BacktestState from freqtrade.enums.rpcmessagetype import RPCMessageType from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode from freqtrade.enums.selltype import SellType diff --git a/freqtrade/enums/backteststate.py b/freqtrade/enums/backteststate.py new file mode 100644 index 000000000..490814497 --- /dev/null +++ b/freqtrade/enums/backteststate.py @@ -0,0 +1,15 @@ +from enum import Enum + + +class BacktestState(Enum): + """ + Bot application states + """ + STARTUP = 1 + DATALOAD = 2 + ANALYZE = 3 + CONVERT = 4 + BACKTEST = 5 + + def __str__(self): + return f"{self.name.lower()}" diff --git a/freqtrade/enums/runmode.py b/freqtrade/enums/runmode.py index 7826d1d0c..6545aaec7 100644 --- a/freqtrade/enums/runmode.py +++ b/freqtrade/enums/runmode.py @@ -14,6 +14,7 @@ class RunMode(Enum): UTIL_EXCHANGE = "util_exchange" UTIL_NO_EXCHANGE = "util_no_exchange" PLOT = "plot" + WEBSERVER = "webserver" OTHER = "other" diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 235f03269..91b278077 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -551,7 +551,7 @@ class Exchange: amount_reserve_percent = 1.0 + self._config.get('amount_reserve_percent', DEFAULT_AMOUNT_RESERVE_PERCENT) amount_reserve_percent = ( - amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5 + amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5 ) # it should not be more than 50% amount_reserve_percent = max(min(amount_reserve_percent, 1.5), 1) @@ -578,7 +578,7 @@ class Exchange: 'side': side, 'remaining': _amount, 'datetime': arrow.utcnow().isoformat(), - 'timestamp': int(arrow.utcnow().int_timestamp * 1000), + 'timestamp': arrow.utcnow().int_timestamp * 1000, 'status': "closed" if ordertype == "market" else "open", 'fee': None, 'info': {} @@ -999,94 +999,64 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e - def get_buy_rate(self, pair: str, refresh: bool) -> float: + def get_rate(self, pair: str, refresh: bool, side: str) -> float: """ - Calculates bid target between current ask price and last price + Calculates bid/ask target + bid rate - between current ask price and last price + ask rate - either using ticker bid or first bid based on orderbook + or remain static in any other case since it's not updating. :param pair: Pair to get rate for :param refresh: allow cached data + :param side: "buy" or "sell" :return: float: Price :raises PricingError if orderbook price could not be determined. """ + cache_rate: TTLCache = self._buy_rate_cache if side == "buy" else self._sell_rate_cache + [strat_name, name] = ['bid_strategy', 'Buy'] if side == "buy" else ['ask_strategy', 'Sell'] + if not refresh: - rate = self._buy_rate_cache.get(pair) + rate = cache_rate.get(pair) # Check if cache has been invalidated if rate: - logger.debug(f"Using cached buy rate for {pair}.") + logger.debug(f"Using cached {side} rate for {pair}.") return rate - bid_strategy = self._config.get('bid_strategy', {}) - if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): + conf_strategy = self._config.get(strat_name, {}) - order_book_top = bid_strategy.get('order_book_top', 1) + if conf_strategy.get('use_order_book', False) and ('use_order_book' in conf_strategy): + + order_book_top = conf_strategy.get('order_book_top', 1) order_book = self.fetch_l2_order_book(pair, order_book_top) logger.debug('order_book %s', order_book) # top 1 = index 0 try: - rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] + rate = order_book[f"{conf_strategy['price_side']}s"][order_book_top - 1][0] except (IndexError, KeyError) as e: logger.warning( - "Buy Price from orderbook could not be determined." - f"Orderbook: {order_book}" - ) - raise PricingError from e - logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side " - f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}") - used_rate = rate_from_l2 - else: - logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") - ticker = self.fetch_ticker(pair) - ticker_rate = ticker[bid_strategy['price_side']] - if ticker['last'] and ticker_rate > ticker['last']: - balance = bid_strategy['ask_last_balance'] - ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) - used_rate = ticker_rate - - self._buy_rate_cache[pair] = used_rate - - return used_rate - - def get_sell_rate(self, pair: str, refresh: bool) -> float: - """ - Get sell rate - either using ticker bid or first bid based on orderbook - or remain static in any other case since it's not updating. - :param pair: Pair to get rate for - :param refresh: allow cached data - :return: Bid rate - :raises PricingError if price could not be determined. - """ - if not refresh: - rate = self._sell_rate_cache.get(pair) - # Check if cache has been invalidated - if rate: - logger.debug(f"Using cached sell rate for {pair}.") - return rate - - ask_strategy = self._config.get('ask_strategy', {}) - if ask_strategy.get('use_order_book', False): - logger.debug( - f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." - ) - order_book_top = ask_strategy.get('order_book_top', 1) - order_book = self.fetch_l2_order_book(pair, order_book_top) - try: - rate = order_book[f"{ask_strategy['price_side']}s"][order_book_top - 1][0] - except (IndexError, KeyError) as e: - logger.warning( - f"Sell Price at location {order_book_top} from orderbook could not be " + f"{name} Price at location {order_book_top} from orderbook could not be " f"determined. Orderbook: {order_book}" ) raise PricingError from e + + logger.info(f"{name} price from orderbook {conf_strategy['price_side'].capitalize()}" + f"side - top {order_book_top} order book {side} rate {rate:.8f}") else: + logger.info(f"Using Last {conf_strategy['price_side'].capitalize()} / Last Price") ticker = self.fetch_ticker(pair) - ticker_rate = ticker[ask_strategy['price_side']] - if ticker['last'] and ticker_rate < ticker['last']: - balance = ask_strategy.get('bid_last_balance', 0.0) - ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last']) + ticker_rate = ticker[conf_strategy['price_side']] + if ticker['last']: + if side == 'buy' and ticker_rate > ticker['last']: + balance = conf_strategy['ask_last_balance'] + ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) + elif side == 'sell' and ticker_rate < ticker['last']: + balance = conf_strategy.get('bid_last_balance', 0.0) + ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last']) rate = ticker_rate if rate is None: - raise PricingError(f"Sell-Rate for {pair} was empty.") - self._sell_rate_cache[pair] = rate + raise PricingError(f"{name}-Rate for {pair} was empty.") + cache_rate[pair] = rate + return rate # Fee handling @@ -1318,8 +1288,8 @@ class Exchange: self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache ohlcv_df = ohlcv_to_dataframe( - ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) + ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) results_df[(pair, timeframe)] = ohlcv_df if cache: self._klines[(pair, timeframe)] = ohlcv_df diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 852d5bf78..8d3b24b10 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -424,16 +424,10 @@ class FreqtradeBot(LoggingMixin): if buy and not sell: stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge) - if not stake_amount: - logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.") - return False - - logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: " - f"{stake_amount} ...") bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {}) if ((bid_check_dom.get('enabled', False)) and - (bid_check_dom.get('bids_to_ask_delta', 0) > 0)): + (bid_check_dom.get('bids_to_ask_delta', 0) > 0)): if self._check_depth_of_market_buy(pair, bid_check_dom): return self.execute_buy(pair, stake_amount, buy_signal_name=buy_signal_name) else: @@ -481,20 +475,29 @@ class FreqtradeBot(LoggingMixin): buy_limit_requested = price else: # Calculate price - buy_limit_requested = self.exchange.get_buy_rate(pair, True) + buy_limit_requested = self.exchange.get_rate(pair, refresh=True, side="buy") if not buy_limit_requested: raise PricingError('Could not determine buy price.') min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, buy_limit_requested, self.strategy.stoploss) - if min_stake_amount is not None and min_stake_amount > stake_amount: - logger.warning( - f"Can't open a new trade for {pair}: stake amount " - f"is too small ({stake_amount} < {min_stake_amount})" - ) + + if not self.edge: + max_stake_amount = self.wallets.get_available_stake_amount() + stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount, + default_retval=stake_amount)( + pair=pair, current_time=datetime.now(timezone.utc), + current_rate=buy_limit_requested, proposed_stake=stake_amount, + min_stake=min_stake_amount, max_stake=max_stake_amount) + stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount) + + if not stake_amount: return False + logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: " + f"{stake_amount} ...") + amount = stake_amount / buy_limit_requested order_type = self.strategy.order_types['buy'] if forcebuy: @@ -607,7 +610,7 @@ class FreqtradeBot(LoggingMixin): """ Sends rpc notification when a buy cancel occurred. """ - current_rate = self.exchange.get_buy_rate(trade.pair, False) + current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="buy") msg = { 'trade_id': trade.id, @@ -693,7 +696,7 @@ class FreqtradeBot(LoggingMixin): (buy, sell, _) = self.strategy.get_signal(trade.pair, self.strategy.timeframe, analyzed_df) logger.debug('checking sell') - sell_rate = self.exchange.get_sell_rate(trade.pair, True) + sell_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell") if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True @@ -1130,7 +1133,8 @@ class FreqtradeBot(LoggingMixin): profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) # Use cached rates here - it was updated seconds ago. - current_rate = self.exchange.get_sell_rate(trade.pair, False) if not fill else None + current_rate = self.exchange.get_rate( + trade.pair, refresh=False, side="sell") if not fill else None profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" @@ -1175,7 +1179,7 @@ class FreqtradeBot(LoggingMixin): profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - current_rate = self.exchange.get_sell_rate(trade.pair, False) + current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="sell") profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 967f08299..6f439866b 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -8,6 +8,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Iterator, List from typing.io import IO +from urllib.parse import urlparse import rapidjson @@ -214,3 +215,16 @@ def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]: """ for chunk in range(0, len(lst), n): yield (lst[chunk:chunk + n]) + + +def parse_db_uri_for_logging(uri: str): + """ + Helper method to parse the DB URI and return the same DB URI with the password censored + if it contains it. Otherwise, return the DB URI unchanged + :param uri: DB URI to parse for logging + """ + parsed_db_uri = urlparse(uri) + if not parsed_db_uri.netloc: # No need for censoring as no password was provided + return uri + pwd = parsed_db_uri.netloc.split(':')[1].split('@')[0] + return parsed_db_uri.geturl().replace(f':{pwd}@', ':*****@') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 35e524f69..7fdc70e70 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -17,10 +17,11 @@ from freqtrade.data import history from freqtrade.data.btanalysis import trade_list_to_dataframe from freqtrade.data.converter import trim_dataframes from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums import SellType +from freqtrade.enums import BacktestState, SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.mixins import LoggingMixin +from freqtrade.optimize.bt_progress import BTProgress from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_stats) from freqtrade.persistence import LocalTrade, PairLocks, Trade @@ -58,6 +59,7 @@ class Backtesting: LoggingMixin.show_output = False self.config = config + self.results: Optional[Dict[str, Any]] = None # Reset keys for backtesting remove_credentials(self.config) @@ -117,6 +119,10 @@ class Backtesting: # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) + self.exchange.validate_required_startup_candles(self.required_startup, self.timeframe) + + self.progress = BTProgress() + self.abort = False def __del__(self): LoggingMixin.show_output = True @@ -129,6 +135,8 @@ class Backtesting: """ self.strategy: IStrategy = strategy strategy.dp = self.dataprovider + # Attach Wallets to Strategy baseclass + IStrategy.wallets = self.wallets # Set stoploss_on_exchange to false for backtesting, # since a "perfect" stoploss-sell is assumed anyway # And the regular "stoploss" function would not apply to that case @@ -145,6 +153,8 @@ class Backtesting: Loads backtest data and returns the data combined with the timerange as tuple. """ + self.progress.init_step(BacktestState.DATALOAD, 1) + timerange = TimeRange.parse_timerange(None if self.config.get( 'timerange') is None else str(self.config.get('timerange'))) @@ -168,6 +178,7 @@ class Backtesting: timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe), self.required_startup, min_date) + self.progress.set_new_value(1) return data, timerange def prepare_backtest(self, enable_protections): @@ -182,6 +193,15 @@ class Backtesting: self.rejected_trades = 0 self.dataprovider.clear_cache() + def check_abort(self): + """ + Check if abort was requested, raise DependencyException if that's the case + Only applies to Interactive backtest mode (webserver mode) + """ + if self.abort: + self.abort = False + raise DependencyException("Stop requested") + def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]: """ Helper function to convert a processed dataframes into lists for performance reasons. @@ -192,8 +212,12 @@ class Backtesting: # and eventually change the constants for indexes at the top headers = ['date', 'buy', 'open', 'close', 'sell', 'low', 'high', 'buy_signal_name'] data: Dict = {} + self.progress.init_step(BacktestState.CONVERT, len(processed)) + # Create dict with data for pair, pair_data in processed.items(): + self.check_abort() + self.progress.increment() if not pair_data.empty: pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist @@ -314,7 +338,18 @@ class Backtesting: stake_amount = self.wallets.get_trade_stake_amount(pair, None) except DependencyException: return None - min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05) + + min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05) or 0 + max_stake_amount = self.wallets.get_available_stake_amount() + + stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount, + default_retval=stake_amount)( + pair=pair, current_time=row[DATE_IDX].to_pydatetime(), current_rate=row[OPEN_IDX], + proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount) + stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount) + + if not stake_amount: + return None order_type = self.strategy.order_types['buy'] time_in_force = self.strategy.order_time_in_force['sell'] @@ -407,10 +442,13 @@ class Backtesting: open_trades: Dict[str, List[LocalTrade]] = defaultdict(list) open_trade_count = 0 + self.progress.init_step(BacktestState.BACKTEST, int( + (end_date - start_date) / timedelta(minutes=self.timeframe_min))) + # Loop timerange and get candle for each pair at that point in time while tmp <= end_date: open_trade_count_start = open_trade_count - + self.check_abort() for i, pair in enumerate(data): row_index = indexes[pair] try: @@ -466,6 +504,7 @@ class Backtesting: self.protections.global_stop(tmp) # Move time one configured time_interval ahead. + self.progress.increment() tmp += timedelta(minutes=self.timeframe_min) trades += self.handle_left_open(open_trades, data=data) @@ -481,6 +520,8 @@ class Backtesting: } def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange): + self.progress.init_step(BacktestState.ANALYZE, 0) + logger.info("Running backtesting for Strategy %s", strat.get_strategy_name()) backtest_start_time = datetime.now(timezone.utc) self._set_strategy(strat) @@ -507,6 +548,7 @@ class Backtesting: "No data left after adjusting for startup candles.") min_date, max_date = history.get_timerange(preprocessed) + logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} ' f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} ' f'({(max_date - min_date).days} days).') @@ -541,11 +583,12 @@ class Backtesting: for strat in self.strategylist: min_date, max_date = self.backtest_one_strategy(strat, data, timerange) if len(self.strategylist) > 0: - stats = generate_backtest_stats(data, self.all_results, - min_date=min_date, max_date=max_date) + + self.results = generate_backtest_stats(data, self.all_results, + min_date=min_date, max_date=max_date) if self.config.get('export', 'none') == 'trades': - store_backtest_stats(self.config['exportfilename'], stats) + store_backtest_stats(self.config['exportfilename'], self.results) # Show backtest results - show_backtest_results(self.config, stats) + show_backtest_results(self.config, self.results) diff --git a/freqtrade/optimize/bt_progress.py b/freqtrade/optimize/bt_progress.py new file mode 100644 index 000000000..d295956c7 --- /dev/null +++ b/freqtrade/optimize/bt_progress.py @@ -0,0 +1,33 @@ +from freqtrade.enums import BacktestState + + +class BTProgress: + _action: BacktestState = BacktestState.STARTUP + _progress: float = 0 + _max_steps: float = 0 + + def __init__(self): + pass + + def init_step(self, action: BacktestState, max_steps: float): + self._action = action + self._max_steps = max_steps + self._proress = 0 + + def set_new_value(self, new_value: float): + self._progress = new_value + + def increment(self): + self._progress += 1 + + @property + def progress(self): + """ + Get progress as ratio, capped to be between 0 and 1 (to avoid small calculation errors). + """ + return max(min(round(self._progress / self._max_steps, 5) + if self._max_steps > 0 else 0, 1), 0) + + @property + def action(self): + return str(self._action) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index c2b2b93cb..80ae8886e 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -12,7 +12,6 @@ from math import ceil from pathlib import Path from typing import Any, Dict, List, Optional -import numpy as np import progressbar import rapidjson from colorama import Fore, Style @@ -20,16 +19,16 @@ from colorama import init as colorama_init from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects from pandas import DataFrame -from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN +from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange -from freqtrade.misc import file_dump_json, plural +from freqtrade.misc import deep_merge_dicts, file_dump_json, plural from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 -from freqtrade.optimize.hyperopt_tools import HyperoptTools +from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver @@ -78,8 +77,11 @@ class Hyperopt: if not self.config.get('hyperopt'): self.custom_hyperopt = HyperOptAuto(self.config) + self.auto_hyperopt = True else: self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) + self.auto_hyperopt = False + self.backtesting._set_strategy(self.backtesting.strategylist[0]) self.custom_hyperopt.strategy = self.backtesting.strategy @@ -163,13 +165,9 @@ class Hyperopt: While not a valid json object - this allows appending easily. :param epoch: result dictionary for this epoch. """ - def default_parser(x): - if isinstance(x, np.integer): - return int(x) - return str(x) - + epoch[FTHYPT_FILEVERSION] = 2 with self.results_file.open('a') as f: - rapidjson.dump(epoch, f, default=default_parser, + rapidjson.dump(epoch, f, default=hyperopt_serializer, number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN) f.write("\n") @@ -201,6 +199,25 @@ class Hyperopt: return result + def _get_no_optimize_details(self) -> Dict[str, Any]: + """ + Get non-optimized parameters + """ + result: Dict[str, Any] = {} + strategy = self.backtesting.strategy + if not HyperoptTools.has_space(self.config, 'roi'): + result['roi'] = {str(k): v for k, v in strategy.minimal_roi.items()} + if not HyperoptTools.has_space(self.config, 'stoploss'): + result['stoploss'] = {'stoploss': strategy.stoploss} + if not HyperoptTools.has_space(self.config, 'trailing'): + result['trailing'] = { + 'trailing_stop': strategy.trailing_stop, + 'trailing_stop_positive': strategy.trailing_stop_positive, + 'trailing_stop_positive_offset': strategy.trailing_stop_positive_offset, + 'trailing_only_offset_is_reached': strategy.trailing_only_offset_is_reached, + } + return result + def print_results(self, results) -> None: """ Log results if it is better than any previous evaluation @@ -310,7 +327,8 @@ class Hyperopt: results_explanation = HyperoptTools.format_results_explanation_string( strat_stats, self.config['stake_currency']) - not_optimized = self.backtesting.strategy.get_params_dict() + not_optimized = self.backtesting.strategy.get_no_optimize_params() + not_optimized = deep_merge_dicts(not_optimized, self._get_no_optimize_details()) trade_count = strat_stats['total_trades'] total_profit = strat_stats['profit_total'] @@ -324,7 +342,8 @@ class Hyperopt: loss = self.calculate_loss(results=backtesting_results['results'], trade_count=trade_count, min_date=min_date, max_date=max_date, - config=self.config, processed=processed) + config=self.config, processed=processed, + backtest_stats=strat_stats) return { 'loss': loss, 'params_dict': params_dict, @@ -469,6 +488,12 @@ class Hyperopt: f"saved to '{self.results_file}'.") if self.current_best_epoch: + if self.auto_hyperopt: + HyperoptTools.try_export_params( + self.config, + self.backtesting.strategy.get_strategy_name(), + self.current_best_epoch) + HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, self.print_json) else: diff --git a/freqtrade/optimize/hyperopt_loss_interface.py b/freqtrade/optimize/hyperopt_loss_interface.py index b5aa588b2..ac8239b75 100644 --- a/freqtrade/optimize/hyperopt_loss_interface.py +++ b/freqtrade/optimize/hyperopt_loss_interface.py @@ -5,7 +5,7 @@ This module defines the interface for the loss-function for hyperopt from abc import ABC, abstractmethod from datetime import datetime -from typing import Dict +from typing import Any, Dict from pandas import DataFrame @@ -22,6 +22,7 @@ class IHyperOptLoss(ABC): def hyperopt_loss_function(results: DataFrame, trade_count: int, min_date: datetime, max_date: datetime, config: Dict, processed: Dict[str, DataFrame], + backtest_stats: Dict[str, Any], *args, **kwargs) -> float: """ Objective function, returns smaller number for better results diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 9eee42a8d..439016c14 100755 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -1,23 +1,82 @@ import io import logging +from copy import deepcopy +from datetime import datetime, timezone from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional +import numpy as np import rapidjson import tabulate from colorama import Fore, Style from pandas import isna, json_normalize +from freqtrade.constants import FTHYPT_FILEVERSION, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException -from freqtrade.misc import round_coin_value, round_dict +from freqtrade.misc import deep_merge_dicts, round_coin_value, round_dict, safe_value_fallback2 logger = logging.getLogger(__name__) +NON_OPT_PARAM_APPENDIX = " # value loaded from strategy" + + +def hyperopt_serializer(x): + if isinstance(x, np.integer): + return int(x) + if isinstance(x, np.bool_): + return bool(x) + + return str(x) + class HyperoptTools(): + @staticmethod + def get_strategy_filename(config: Dict, strategy_name: str) -> Optional[Path]: + """ + Get Strategy-location (filename) from strategy_name + """ + from freqtrade.resolvers.strategy_resolver import StrategyResolver + directory = Path(config.get('strategy_path', config['user_data_dir'] / USERPATH_STRATEGIES)) + strategy_objs = StrategyResolver.search_all_objects(directory, False) + strategies = [s for s in strategy_objs if s['name'] == strategy_name] + if strategies: + strategy = strategies[0] + + return Path(strategy['location']) + return None + + @staticmethod + def export_params(params, strategy_name: str, filename: Path): + """ + Generate files + """ + final_params = deepcopy(params['params_not_optimized']) + final_params = deep_merge_dicts(params['params_details'], final_params) + final_params = { + 'strategy_name': strategy_name, + 'params': final_params, + 'ft_stratparam_v': 1, + 'export_time': datetime.now(timezone.utc), + } + logger.info(f"Dumping parameters to {filename}") + rapidjson.dump(final_params, filename.open('w'), indent=2, + default=hyperopt_serializer, + number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN + ) + + @staticmethod + def try_export_params(config: Dict[str, Any], strategy_name: str, params: Dict): + if params.get(FTHYPT_FILEVERSION, 1) >= 2 and not config.get('disableparamexport', False): + # Export parameters ... + fn = HyperoptTools.get_strategy_filename(config, strategy_name) + if fn: + HyperoptTools.export_params(params, strategy_name, fn.with_suffix('.json')) + else: + logger.warning("Strategy not found, not exporting parameter file.") + @staticmethod def has_space(config: Dict[str, Any], space: str) -> bool: """ @@ -99,9 +158,9 @@ class HyperoptTools(): non_optimized) HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:", non_optimized) - HyperoptTools._params_pretty_print(params, 'roi', "ROI table:") - HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:") - HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:") + HyperoptTools._params_pretty_print(params, 'roi', "ROI table:", non_optimized) + HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:", non_optimized) + HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:", non_optimized) @staticmethod def _params_update_for_json(result_dict, params, non_optimized, space: str) -> None: @@ -127,23 +186,34 @@ class HyperoptTools(): def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None: if space in params or space in non_optimized: space_params = HyperoptTools._space_params(params, space, 5) + no_params = HyperoptTools._space_params(non_optimized, space, 5) + appendix = '' + if not space_params and not no_params: + # No parameters - don't print + return + if not space_params: + # Not optimized parameters - append string + appendix = NON_OPT_PARAM_APPENDIX + result = f"\n# {header}\n" - if space == 'stoploss': - result += f"stoploss = {space_params.get('stoploss')}" - elif space == 'roi': + if space == "stoploss": + stoploss = safe_value_fallback2(space_params, no_params, space, space) + result += (f"stoploss = {stoploss}{appendix}") + + elif space == "roi": + result = result[:-1] + f'{appendix}\n' minimal_roi_result = rapidjson.dumps({ - str(k): v for k, v in space_params.items() + str(k): v for k, v in (space_params or no_params).items() }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE) result += f"minimal_roi = {minimal_roi_result}" - elif space == 'trailing': - - for k, v in space_params.items(): - result += f'{k} = {v}\n' + elif space == "trailing": + for k, v in (space_params or no_params).items(): + result += f"{k} = {v}{appendix}\n" else: - no_params = HyperoptTools._space_params(non_optimized, space, 5) + # Buy / sell parameters - result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}" + result += f"{space}_params = {HyperoptTools._pprint_dict(space_params, no_params)}" result = result.replace("\n", "\n ") print(result) @@ -157,7 +227,7 @@ class HyperoptTools(): return {} @staticmethod - def _pprint(params, non_optimized, indent: int = 4): + def _pprint_dict(params, non_optimized, indent: int = 4): """ Pretty-print hyperopt results (based on 2 dicts - with add. comment) """ @@ -169,7 +239,7 @@ class HyperoptTools(): result += " " * indent + f'"{k}": ' result += f'"{param}",' if isinstance(param, str) else f'{param},' if k in non_optimized: - result += " # value loaded from strategy" + result += NON_OPT_PARAM_APPENDIX result += "\n" result += '}' return result diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 79208c5e9..eefacbbab 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -229,8 +229,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]: winning_trades = results.loc[results['profit_ratio'] > 0] draw_trades = results.loc[results['profit_ratio'] == 0] losing_trades = results.loc[results['profit_ratio'] < 0] - zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) & - (results['sell_reason'] == 'trailing_stop_loss')]) holding_avg = (timedelta(minutes=round(results['trade_duration'].mean())) if not results.empty else timedelta()) @@ -249,7 +247,6 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]: 'winner_holding_avg_s': winner_holding_avg.total_seconds(), 'loser_holding_avg': loser_holding_avg, 'loser_holding_avg_s': loser_holding_avg.total_seconds(), - 'zero_duration_trades': zero_duration_trades, } @@ -264,6 +261,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: 'winning_days': 0, 'draw_days': 0, 'losing_days': 0, + 'daily_profit_list': [], } daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum() daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10) @@ -274,6 +272,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: winning_days = sum(daily_profit > 0) draw_days = sum(daily_profit == 0) losing_days = sum(daily_profit < 0) + daily_profit_list = [(str(idx.date()), val) for idx, val in daily_profit.iteritems()] return { 'backtest_best_day': best_rel, @@ -283,6 +282,7 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]: 'winning_days': winning_days, 'draw_days': draw_days, 'losing_days': losing_days, + 'daily_profit': daily_profit_list, } @@ -325,8 +325,9 @@ def generate_strategy_stats(btdata: Dict[str, DataFrame], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None - results['open_timestamp'] = results['open_date'].astype(int64) // 1e6 - results['close_timestamp'] = results['close_date'].astype(int64) // 1e6 + if not results.empty: + results['open_timestamp'] = results['open_date'].view(int64) // 1e6 + results['close_timestamp'] = results['close_date'].view(int64) // 1e6 backtest_days = (max_date - min_date).days strat_stats = { @@ -542,14 +543,6 @@ def text_table_add_metrics(strat_results: Dict) -> str: # Newly added fields should be ignored if they are missing in strat_results. hyperopt-show # command stores these results and newer version of freqtrade must be able to handle old # results with missing new fields. - zero_duration_trades = '--' - - if 'zero_duration_trades' in strat_results: - zero_duration_trades_per = \ - 100.0 / strat_results['total_trades'] * strat_results['zero_duration_trades'] - zero_duration_trades = f'{zero_duration_trades_per:.2f}% ' \ - f'({strat_results["zero_duration_trades"]})' - metrics = [ ('Backtesting from', strat_results['backtest_start']), ('Backtesting to', strat_results['backtest_end']), @@ -585,7 +578,6 @@ def text_table_add_metrics(strat_results: Dict) -> str: f"{strat_results['draw_days']} / {strat_results['losing_days']}"), ('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"), ('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"), - ('Zero Duration Trades', zero_duration_trades), ('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')), ('', ''), # Empty line to improve readability diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 8d77ac27a..058e892ec 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -804,6 +804,19 @@ class Trade(_DECL_BASE, LocalTrade): Trade.is_open.is_(False), ]).all() + @staticmethod + def get_total_closed_profit() -> float: + """ + Retrieves total realized profit + """ + if Trade.use_db: + total_profit = Trade.query.with_entities( + func.sum(Trade.close_profit_abs)).filter(Trade.is_open.is_(False)).scalar() + else: + total_profit = sum( + t.close_profit_abs for t in LocalTrade.get_trades_proxy(is_open=False)) + return total_profit or 0 + @staticmethod def total_open_trades_stakes() -> float: """ diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 8f623b062..dc5cab31e 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -27,6 +27,7 @@ class AgeFilter(IPairList): super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) self._min_days_listed = pairlistconfig.get('min_days_listed', 10) + self._max_days_listed = pairlistconfig.get('max_days_listed', None) if self._min_days_listed < 1: raise OperationalException("AgeFilter requires min_days_listed to be >= 1") @@ -34,6 +35,12 @@ class AgeFilter(IPairList): raise OperationalException("AgeFilter requires min_days_listed to not exceed " "exchange max request size " f"({exchange.ohlcv_candle_limit('1d')})") + if self._max_days_listed and self._max_days_listed <= self._min_days_listed: + raise OperationalException("AgeFilter max_days_listed <= min_days_listed not permitted") + if self._max_days_listed and self._max_days_listed > exchange.ohlcv_candle_limit('1d'): + raise OperationalException("AgeFilter requires max_days_listed to not exceed " + "exchange max request size " + f"({exchange.ohlcv_candle_limit('1d')})") @property def needstickers(self) -> bool: @@ -48,8 +55,13 @@ class AgeFilter(IPairList): """ Short whitelist method description - used for startup-messages """ - return (f"{self.name} - Filtering pairs with age less than " - f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.") + return ( + f"{self.name} - Filtering pairs with age less than " + f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}" + ) + (( + " or more than " + f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" + ) if self._max_days_listed else '') def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ @@ -61,9 +73,12 @@ class AgeFilter(IPairList): if not needed_pairs: return pairlist + since_days = -( + self._max_days_listed if self._max_days_listed else self._min_days_listed + ) - 1 since_ms = int(arrow.utcnow() .floor('day') - .shift(days=-self._min_days_listed - 1) + .shift(days=since_days) .float_timestamp) * 1000 candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: @@ -86,14 +101,22 @@ class AgeFilter(IPairList): return True if daily_candles is not None: - if len(daily_candles) >= self._min_days_listed: + if ( + len(daily_candles) >= self._min_days_listed + and (not self._max_days_listed or len(daily_candles) <= self._max_days_listed) + ): # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol - self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000 + self._symbolsChecked[pair] = arrow.utcnow().int_timestamp * 1000 return True else: - self.log_once(f"Removed {pair} from whitelist, because age " - f"{len(daily_candles)} is less than {self._min_days_listed} " - f"{plural(self._min_days_listed, 'day')}", logger.info) + self.log_once(( + f"Removed {pair} from whitelist, because age " + f"{len(daily_candles)} is less than {self._min_days_listed} " + f"{plural(self._min_days_listed, 'day')}" + ) + (( + " or more than " + f"{self._max_days_listed} {plural(self._max_days_listed, 'day')}" + ) if self._max_days_listed else ''), logger.info) return False return False diff --git a/freqtrade/plugins/pairlist/OffsetFilter.py b/freqtrade/plugins/pairlist/OffsetFilter.py new file mode 100644 index 000000000..573a573a6 --- /dev/null +++ b/freqtrade/plugins/pairlist/OffsetFilter.py @@ -0,0 +1,54 @@ +""" +Offset pair list filter +""" +import logging +from typing import Any, Dict, List + +from freqtrade.exceptions import OperationalException +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class OffsetFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + self._offset = pairlistconfig.get('offset', 0) + + if self._offset < 0: + raise OperationalException("OffsetFilter requires offset to be >= 0") + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requires tickers, an empty Dict is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short whitelist method description - used for startup-messages + """ + return f"{self.name} - Offseting pairs by {self._offset}." + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the whitelist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new whitelist + """ + if self._offset > len(pairlist): + self.log_once(f"Offset of {self._offset} is larger than " + + f"pair count of {len(pairlist)}", logger.warning) + pairs = pairlist[self._offset:] + self.log_once(f"Searching {len(pairs)} pairs: {pairs}", logger.info) + return pairs diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 5ae8e3e9f..9383e5d06 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -69,10 +69,10 @@ class VolatilityFilter(IPairList): """ needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .float_timestamp) * 1000 + since_ms = (arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .int_timestamp) * 1000 # Get all candles candles = {} if needed_pairs: diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 8eff137b0..d6b8aaaa3 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -6,9 +6,12 @@ Provides dynamic pair list based on trade volumes import logging from typing import Any, Dict, List +import arrow from cachetools.ttl import TTLCache from freqtrade.exceptions import OperationalException +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.misc import format_ms_time from freqtrade.plugins.pairlist.IPairList import IPairList @@ -36,6 +39,35 @@ class VolumePairList(IPairList): self._min_value = self._pairlistconfig.get('min_value', 0) self._refresh_period = self._pairlistconfig.get('refresh_period', 1800) self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period) + self._lookback_days = self._pairlistconfig.get('lookback_days', 0) + self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d') + self._lookback_period = self._pairlistconfig.get('lookback_period', 0) + + if (self._lookback_days > 0) & (self._lookback_period > 0): + raise OperationalException( + 'Ambigous configuration: lookback_days and lookback_period both set in pairlist ' + 'config. Please set lookback_days only or lookback_period and lookback_timeframe ' + 'and restart the bot.' + ) + + # overwrite lookback timeframe and days when lookback_days is set + if self._lookback_days > 0: + self._lookback_timeframe = '1d' + self._lookback_period = self._lookback_days + + # get timeframe in minutes and seconds + self._tf_in_min = timeframe_to_minutes(self._lookback_timeframe) + self._tf_in_sec = self._tf_in_min * 60 + + # wether to use range lookback or not + self._use_range = (self._tf_in_min > 0) & (self._lookback_period > 0) + + if self._use_range & (self._refresh_period < self._tf_in_sec): + raise OperationalException( + f'Refresh period of {self._refresh_period} seconds is smaller than one ' + f'timeframe of {self._lookback_timeframe}. Please adjust refresh_period ' + f'to at least {self._tf_in_sec} and restart the bot.' + ) if not self._exchange.exchange_has('fetchTickers'): raise OperationalException( @@ -47,6 +79,13 @@ class VolumePairList(IPairList): raise OperationalException( f'key {self._sort_key} not in {SORT_VALUES}') + if self._lookback_period < 0: + raise OperationalException("VolumeFilter requires lookback_period to be >= 0") + if self._lookback_period > exchange.ohlcv_candle_limit(self._lookback_timeframe): + raise OperationalException("VolumeFilter requires lookback_period to not " + "exceed exchange max request size " + f"({exchange.ohlcv_candle_limit(self._lookback_timeframe)})") + @property def needstickers(self) -> bool: """ @@ -78,7 +117,6 @@ class VolumePairList(IPairList): # Item found - no refresh necessary return pairlist else: - # Use fresh pairlist # Check if pair quote currency equals to the stake currency. filtered_tickers = [ @@ -103,6 +141,60 @@ class VolumePairList(IPairList): # Use the incoming pairlist. filtered_tickers = [v for k, v in tickers.items() if k in pairlist] + # get lookback period in ms, for exchange ohlcv fetch + if self._use_range: + since_ms = int(arrow.utcnow() + .floor('minute') + .shift(minutes=-(self._lookback_period * self._tf_in_min) + - self._tf_in_min) + .int_timestamp) * 1000 + + to_ms = int(arrow.utcnow() + .floor('minute') + .shift(minutes=-self._tf_in_min) + .int_timestamp) * 1000 + + # todo: utc date output for starting date + self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: " + f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " + f"till {format_ms_time(to_ms)}", logger.info) + needed_pairs = [ + (p, self._lookback_timeframe) for p in + [ + s['symbol'] for s in filtered_tickers + ] if p not in self._pair_cache + ] + + # Get all candles + candles = {} + if needed_pairs: + candles = self._exchange.refresh_latest_ohlcv( + needed_pairs, since_ms=since_ms, cache=False + ) + for i, p in enumerate(filtered_tickers): + pair_candles = candles[ + (p['symbol'], self._lookback_timeframe) + ] if (p['symbol'], self._lookback_timeframe) in candles else None + # in case of candle data calculate typical price and quoteVolume for candle + if pair_candles is not None and not pair_candles.empty: + pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] + + pair_candles['close']) / 3 + pair_candles['quoteVolume'] = ( + pair_candles['volume'] * pair_candles['typical_price'] + ) + + # ensure that a rolling sum over the lookback_period is built + # if pair_candles contains more candles than lookback_period + quoteVolume = (pair_candles['quoteVolume'] + .rolling(self._lookback_period) + .sum() + .iloc[-1]) + + # replace quoteVolume with range quoteVolume sum calculated above + filtered_tickers[i]['quoteVolume'] = quoteVolume + else: + filtered_tickers[i]['quoteVolume'] = 0 + if self._min_value > 0: filtered_tickers = [ v for v in filtered_tickers if v[self._sort_key] > self._min_value] diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 8be61166b..a6d1820de 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -62,10 +62,10 @@ class RangeStabilityFilter(IPairList): """ needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._days - 1) - .float_timestamp) * 1000 + since_ms = (arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .int_timestamp) * 1000 # Get all candles candles = {} if needed_pairs: diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index ccd7cea69..1239b78b3 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -53,6 +53,21 @@ class StrategyResolver(IResolver): ) strategy.timeframe = strategy.ticker_interval + if strategy._ft_params_from_file: + # Set parameters from Hyperopt results file + params = strategy._ft_params_from_file + strategy.minimal_roi = params.get('roi', strategy.minimal_roi) + + strategy.stoploss = params.get('stoploss', {}).get('stoploss', strategy.stoploss) + trailing = params.get('trailing', {}) + strategy.trailing_stop = trailing.get('trailing_stop', strategy.trailing_stop) + strategy.trailing_stop_positive = trailing.get('trailing_stop_positive', + strategy.trailing_stop_positive) + strategy.trailing_stop_positive_offset = trailing.get( + 'trailing_stop_positive_offset', strategy.trailing_stop_positive_offset) + strategy.trailing_only_offset_is_reached = trailing.get( + 'trailing_only_offset_is_reached', strategy.trailing_only_offset_is_reached) + # Set attributes # Check if we need to override configuration # (Attribute name, default, subkey) diff --git a/freqtrade/rpc/api_server/api_backtest.py b/freqtrade/rpc/api_server/api_backtest.py new file mode 100644 index 000000000..76b4a8169 --- /dev/null +++ b/freqtrade/rpc/api_server/api_backtest.py @@ -0,0 +1,176 @@ +import asyncio +import logging +from copy import deepcopy + +from fastapi import APIRouter, BackgroundTasks, Depends + +from freqtrade.enums import BacktestState +from freqtrade.exceptions import DependencyException +from freqtrade.rpc.api_server.api_schemas import BacktestRequest, BacktestResponse +from freqtrade.rpc.api_server.deps import get_config +from freqtrade.rpc.api_server.webserver import ApiServer +from freqtrade.rpc.rpc import RPCException + + +logger = logging.getLogger(__name__) + +# Private API, protected by authentication +router = APIRouter() + + +@router.post('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest']) +async def api_start_backtest(bt_settings: BacktestRequest, background_tasks: BackgroundTasks, + config=Depends(get_config)): + """Start backtesting if not done so already""" + if ApiServer._bgtask_running: + raise RPCException('Bot Background task already running') + + btconfig = deepcopy(config) + settings = dict(bt_settings) + # Pydantic models will contain all keys, but non-provided ones are None + for setting in settings.keys(): + if settings[setting] is not None: + btconfig[setting] = settings[setting] + + # Start backtesting + # Initialize backtesting object + def run_backtest(): + from freqtrade.optimize.optimize_reports import generate_backtest_stats + from freqtrade.resolvers import StrategyResolver + asyncio.set_event_loop(asyncio.new_event_loop()) + try: + # Reload strategy + lastconfig = ApiServer._bt_last_config + strat = StrategyResolver.load_strategy(btconfig) + + if ( + not ApiServer._bt + or lastconfig.get('timeframe') != strat.timeframe + or lastconfig.get('dry_run_wallet') != btconfig.get('dry_run_wallet', 0) + ): + from freqtrade.optimize.backtesting import Backtesting + ApiServer._bt = Backtesting(btconfig) + + # Only reload data if timeframe or timerange changed. + if ( + not ApiServer._bt_data + or not ApiServer._bt_timerange + or lastconfig.get('timerange') != btconfig['timerange'] + or lastconfig.get('stake_amount') != btconfig.get('stake_amount') + or lastconfig.get('enable_protections') != btconfig.get('enable_protections') + or lastconfig.get('protections') != btconfig.get('protections', []) + or lastconfig.get('timeframe') != strat.timeframe + ): + lastconfig['timerange'] = btconfig['timerange'] + lastconfig['protections'] = btconfig.get('protections', []) + lastconfig['enable_protections'] = btconfig.get('enable_protections') + lastconfig['dry_run_wallet'] = btconfig.get('dry_run_wallet') + lastconfig['timeframe'] = strat.timeframe + ApiServer._bt_data, ApiServer._bt_timerange = ApiServer._bt.load_bt_data() + + ApiServer._bt.abort = False + min_date, max_date = ApiServer._bt.backtest_one_strategy( + strat, ApiServer._bt_data, ApiServer._bt_timerange) + ApiServer._bt.results = generate_backtest_stats( + ApiServer._bt_data, ApiServer._bt.all_results, + min_date=min_date, max_date=max_date) + logger.info("Backtest finished.") + + except DependencyException as e: + logger.info(f"Backtesting caused an error: {e}") + pass + finally: + ApiServer._bgtask_running = False + + background_tasks.add_task(run_backtest) + ApiServer._bgtask_running = True + + return { + "status": "running", + "running": True, + "progress": 0, + "step": str(BacktestState.STARTUP), + "status_msg": "Backtest started", + } + + +@router.get('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest']) +def api_get_backtest(): + """ + Get backtesting result. + Returns Result after backtesting has been ran. + """ + from freqtrade.persistence import LocalTrade + if ApiServer._bgtask_running: + return { + "status": "running", + "running": True, + "step": ApiServer._bt.progress.action if ApiServer._bt else str(BacktestState.STARTUP), + "progress": ApiServer._bt.progress.progress if ApiServer._bt else 0, + "trade_count": len(LocalTrade.trades), + "status_msg": "Backtest running", + } + + if not ApiServer._bt: + return { + "status": "not_started", + "running": False, + "step": "", + "progress": 0, + "status_msg": "Backtest not yet executed" + } + + return { + "status": "ended", + "running": False, + "status_msg": "Backtest ended", + "step": "finished", + "progress": 1, + "backtest_result": ApiServer._bt.results, + } + + +@router.delete('/backtest', response_model=BacktestResponse, tags=['webserver', 'backtest']) +def api_delete_backtest(): + """Reset backtesting""" + if ApiServer._bgtask_running: + return { + "status": "running", + "running": True, + "step": "", + "progress": 0, + "status_msg": "Backtest running", + } + if ApiServer._bt: + del ApiServer._bt + ApiServer._bt = None + del ApiServer._bt_data + ApiServer._bt_data = None + logger.info("Backtesting reset") + return { + "status": "reset", + "running": False, + "step": "", + "progress": 0, + "status_msg": "Backtest reset", + } + + +@router.get('/backtest/abort', response_model=BacktestResponse, tags=['webserver', 'backtest']) +def api_backtest_abort(): + if not ApiServer._bgtask_running: + return { + "status": "not_running", + "running": False, + "step": "", + "progress": 0, + "status_msg": "Backtest ended", + } + ApiServer._bt.abort = True + return { + "status": "stopping", + "running": False, + "step": "", + "progress": 0, + "status_msg": "Backtest ended", + } diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index a0f1c05a6..c6b6a6d28 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -67,12 +67,16 @@ class Profit(BaseModel): profit_closed_ratio_mean: float profit_closed_percent_sum: float profit_closed_ratio_sum: float + profit_closed_percent: float + profit_closed_ratio: float profit_closed_fiat: float profit_all_coin: float profit_all_percent_mean: float profit_all_ratio_mean: float profit_all_percent_sum: float profit_all_ratio_sum: float + profit_all_percent: float + profit_all_ratio: float profit_all_fiat: float trade_count: int closed_trade_count: int @@ -115,20 +119,21 @@ class ShowConfig(BaseModel): dry_run: bool stake_currency: str stake_amount: Union[float, str] + available_capital: Optional[float] stake_currency_decimals: int max_open_trades: int minimal_roi: Dict[str, Any] - stoploss: float - trailing_stop: bool + stoploss: Optional[float] + trailing_stop: Optional[bool] trailing_stop_positive: Optional[float] trailing_stop_positive_offset: Optional[float] trailing_only_offset_is_reached: Optional[bool] use_custom_stoploss: Optional[bool] - timeframe: str + timeframe: Optional[str] timeframe_ms: int timeframe_min: int exchange: str - strategy: str + strategy: Optional[str] forcebuy_enabled: bool ask_strategy: Dict[str, Any] bid_strategy: Dict[str, Any] @@ -313,3 +318,24 @@ class PairHistory(BaseModel): json_encoders = { datetime: lambda v: v.strftime(DATETIME_PRINT_FORMAT), } + + +class BacktestRequest(BaseModel): + strategy: str + timeframe: Optional[str] + timerange: Optional[str] + max_open_trades: Optional[int] + stake_amount: Optional[Union[float, str]] + enable_protections: bool + dry_run_wallet: Optional[float] + + +class BacktestResponse(BaseModel): + status: str + running: bool + status_msg: str + step: str + progress: float + trade_count: Optional[float] + # TODO: Properly type backtestresult... + backtest_result: Optional[Dict[str, Any]] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 965664028..61d69707e 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -1,3 +1,4 @@ +import logging from copy import deepcopy from pathlib import Path from typing import List, Optional @@ -22,6 +23,8 @@ from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException +logger = logging.getLogger(__name__) + # Public API, requires no auth. router_public = APIRouter() # Private API, protected by authentication @@ -249,7 +252,7 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option pair_interval = sorted(pair_interval, key=lambda x: x[0]) pairs = list({x[0] for x in pair_interval}) - + pairs.sort() result = { 'length': len(pairs), 'pairs': pairs, diff --git a/freqtrade/rpc/api_server/web_ui.py b/freqtrade/rpc/api_server/web_ui.py index a8c737e04..76c8ed8f2 100644 --- a/freqtrade/rpc/api_server/web_ui.py +++ b/freqtrade/rpc/api_server/web_ui.py @@ -18,6 +18,17 @@ async def fallback(): return FileResponse(str(Path(__file__).parent / 'ui/fallback_file.html')) +@router_ui.get('/ui_version', include_in_schema=False) +async def ui_version(): + from freqtrade.commands.deploy_commands import read_ui_version + uibase = Path(__file__).parent / 'ui/installed/' + version = read_ui_version(uibase) + + return { + "version": version if version else "not_installed", + } + + @router_ui.get('/{rest_of_path:path}', include_in_schema=False) async def index_html(rest_of_path: str): """ diff --git a/freqtrade/rpc/api_server/webserver.py b/freqtrade/rpc/api_server/webserver.py index a43d4abe6..235063191 100644 --- a/freqtrade/rpc/api_server/webserver.py +++ b/freqtrade/rpc/api_server/webserver.py @@ -8,6 +8,7 @@ from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware from starlette.responses import JSONResponse +from freqtrade.exceptions import OperationalException from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler @@ -28,17 +29,37 @@ class FTJSONResponse(JSONResponse): class ApiServer(RPCHandler): + __instance = None + __initialized = False + _rpc: RPC + # Backtesting type: Backtesting + _bt = None + _bt_data = None + _bt_timerange = None + _bt_last_config: Dict[str, Any] = {} _has_rpc: bool = False + _bgtask_running: bool = False _config: Dict[str, Any] = {} - def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: - super().__init__(rpc, config) - self._server = None + def __new__(cls, *args, **kwargs): + """ + This class is a singleton. + We'll only have one instance of it around. + """ + if ApiServer.__instance is None: + ApiServer.__instance = object.__new__(cls) + ApiServer.__initialized = False + return ApiServer.__instance - ApiServer._rpc = rpc - ApiServer._has_rpc = True + def __init__(self, config: Dict[str, Any], standalone: bool = False) -> None: ApiServer._config = config + if self.__initialized and (standalone or self._standalone): + return + self._standalone: bool = standalone + self._server = None + ApiServer.__initialized = True + api_config = self._config['api_server'] self.app = FastAPI(title="Freqtrade API", @@ -50,12 +71,33 @@ class ApiServer(RPCHandler): self.start_api() + def add_rpc_handler(self, rpc: RPC): + """ + Attach rpc handler + """ + if not self._has_rpc: + ApiServer._rpc = rpc + ApiServer._has_rpc = True + else: + # This should not happen assuming we didn't mess up. + raise OperationalException('RPC Handler already attached.') + def cleanup(self) -> None: """ Cleanup pending module resources """ - if self._server: + ApiServer._has_rpc = False + del ApiServer._rpc + if self._server and not self._standalone: logger.info("Stopping API Server") self._server.cleanup() + @classmethod + def shutdown(cls): + cls.__initialized = False + del cls.__instance + cls.__instance = None + cls._has_rpc = False + cls._rpc = None + def send_msg(self, msg: Dict[str, str]) -> None: pass @@ -68,6 +110,7 @@ class ApiServer(RPCHandler): def configure_app(self, app: FastAPI, config): from freqtrade.rpc.api_server.api_auth import http_basic_or_jwt_token, router_login + from freqtrade.rpc.api_server.api_backtest import router as api_backtest from freqtrade.rpc.api_server.api_v1 import router as api_v1 from freqtrade.rpc.api_server.api_v1 import router_public as api_v1_public from freqtrade.rpc.api_server.web_ui import router_ui @@ -77,6 +120,9 @@ class ApiServer(RPCHandler): app.include_router(api_v1, prefix="/api/v1", dependencies=[Depends(http_basic_or_jwt_token)], ) + app.include_router(api_backtest, prefix="/api/v1", + dependencies=[Depends(http_basic_or_jwt_token)], + ) app.include_router(router_login, prefix="/api/v1", tags=["auth"]) # UI Router MUST be last! app.include_router(router_ui, prefix='') @@ -125,6 +171,9 @@ class ApiServer(RPCHandler): ) try: self._server = UvicornServer(uvconfig) - self._server.run_in_thread() + if self._standalone: + self._server.run() + else: + self._server.run_in_thread() except Exception: logger.exception("Api server failed to start.") diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 538e95f40..902975fde 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -106,6 +106,7 @@ class RPC: 'stake_currency': config['stake_currency'], 'stake_currency_decimals': decimals_per_coin(config['stake_currency']), 'stake_amount': config['stake_amount'], + 'available_capital': config.get('available_capital'), 'max_open_trades': (config['max_open_trades'] if config['max_open_trades'] != float('inf') else -1), 'minimal_roi': config['minimal_roi'].copy() if 'minimal_roi' in config else {}, @@ -118,9 +119,9 @@ class RPC: 'bot_name': config.get('bot_name', 'freqtrade'), 'timeframe': config.get('timeframe'), 'timeframe_ms': timeframe_to_msecs(config['timeframe'] - ) if 'timeframe' in config else '', + ) if 'timeframe' in config else 0, 'timeframe_min': timeframe_to_minutes(config['timeframe'] - ) if 'timeframe' in config else '', + ) if 'timeframe' in config else 0, 'exchange': config['exchange']['name'], 'strategy': config['strategy'], 'forcebuy_enabled': config.get('forcebuy_enable', False), @@ -153,7 +154,8 @@ class RPC: # calculate profit and send message to user if trade.is_open: try: - current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_rate( + trade.pair, refresh=False, side="sell") except (ExchangeError, PricingError): current_rate = NAN else: @@ -212,7 +214,8 @@ class RPC: for trade in trades: # calculate profit and send message to user try: - current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_rate( + trade.pair, refresh=False, side="sell") except (PricingError, ExchangeError): current_rate = NAN trade_percent = (100 * trade.calc_profit_ratio(current_rate)) @@ -271,10 +274,10 @@ class RPC: 'date': key, 'abs_profit': value["amount"], 'fiat_value': self._fiat_converter.convert_amount( - value['amount'], - stake_currency, - fiat_display_currency - ) if self._fiat_converter else 0, + value['amount'], + stake_currency, + fiat_display_currency + ) if self._fiat_converter else 0, 'trade_count': value["trades"], } for key, value in profit_days.items() @@ -371,7 +374,8 @@ class RPC: else: # Get current rate try: - current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_rate( + trade.pair, refresh=False, side="sell") except (PricingError, ExchangeError): current_rate = NAN profit_ratio = trade.calc_profit_ratio(rate=current_rate) @@ -396,7 +400,12 @@ class RPC: profit_all_coin_sum = round(sum(profit_all_coin), 8) profit_all_ratio_mean = float(mean(profit_all_ratio) if profit_all_ratio else 0.0) + # Doing the sum is not right - overall profit needs to be based on initial capital profit_all_ratio_sum = sum(profit_all_ratio) if profit_all_ratio else 0.0 + starting_balance = self._freqtrade.wallets.get_starting_balance() + profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance + profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance + profit_all_fiat = self._fiat_converter.convert_amount( profit_all_coin_sum, stake_currency, @@ -412,12 +421,16 @@ class RPC: 'profit_closed_ratio_mean': profit_closed_ratio_mean, 'profit_closed_percent_sum': round(profit_closed_ratio_sum * 100, 2), 'profit_closed_ratio_sum': profit_closed_ratio_sum, + 'profit_closed_ratio': profit_closed_ratio_fromstart, + 'profit_closed_percent': round(profit_closed_ratio_fromstart * 100, 2), 'profit_closed_fiat': profit_closed_fiat, 'profit_all_coin': profit_all_coin_sum, 'profit_all_percent_mean': round(profit_all_ratio_mean * 100, 2), 'profit_all_ratio_mean': profit_all_ratio_mean, 'profit_all_percent_sum': round(profit_all_ratio_sum * 100, 2), 'profit_all_ratio_sum': profit_all_ratio_sum, + 'profit_all_ratio': profit_all_ratio_fromstart, + 'profit_all_percent': round(profit_all_ratio_fromstart * 100, 2), 'profit_all_fiat': profit_all_fiat, 'trade_count': len(trades), 'closed_trade_count': len([t for t in trades if not t.is_open]), @@ -541,7 +554,8 @@ class RPC: if not fully_canceled: # Get current rate and execute sell - current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_rate( + trade.pair, refresh=False, side="sell") sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL) self._freqtrade.execute_sell(trade, current_rate, sell_reason) # ---- EOF def _exec_forcesell ---- @@ -761,7 +775,7 @@ class RPC: sell_signals = 0 if has_content: - dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].astype(int64) // 1000 // 1000 + dataframe.loc[:, '__date_ts'] = dataframe.loc[:, 'date'].view(int64) // 1000 // 1000 # Move open to seperate column when signal for easy plotting if 'buy' in dataframe.columns: buy_mask = (dataframe['buy'] == 1) diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index 18ed68041..67842e849 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -1,5 +1,5 @@ """ -This module contains class to manage RPC communications (Telegram, Slack, ...) +This module contains class to manage RPC communications (Telegram, API, ...) """ import logging from typing import Any, Dict, List @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) class RPCManager: """ - Class to manage RPC objects (Telegram, Slack, ...) + Class to manage RPC objects (Telegram, API, ...) """ def __init__(self, freqtrade) -> None: """ Initializes all enabled rpc modules """ @@ -36,15 +36,16 @@ class RPCManager: if config.get('api_server', {}).get('enabled', False): logger.info('Enabling rpc.api_server') from freqtrade.rpc.api_server import ApiServer - - self.registered_modules.append(ApiServer(self._rpc, config)) + apiserver = ApiServer(config) + apiserver.add_rpc_handler(self._rpc) + self.registered_modules.append(apiserver) def cleanup(self) -> None: """ Stops all enabled rpc modules """ logger.info('Cleaning up rpc modules ...') while self.registered_modules: mod = self.registered_modules.pop() - logger.debug('Cleaning up rpc.%s ...', mod.name) + logger.info('Cleaning up rpc.%s ...', mod.name) mod.cleanup() del mod diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index d857e46a9..263a3fc6d 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -24,7 +24,7 @@ from freqtrade.__init__ import __version__ from freqtrade.constants import DUST_PER_COIN from freqtrade.enums import RPCMessageType from freqtrade.exceptions import OperationalException -from freqtrade.misc import chunks, round_coin_value +from freqtrade.misc import chunks, plural, round_coin_value from freqtrade.rpc import RPC, RPCException, RPCHandler @@ -494,11 +494,11 @@ class Telegram(RPCHandler): start_date) profit_closed_coin = stats['profit_closed_coin'] profit_closed_percent_mean = stats['profit_closed_percent_mean'] - profit_closed_percent_sum = stats['profit_closed_percent_sum'] + profit_closed_percent = stats['profit_closed_percent'] profit_closed_fiat = stats['profit_closed_fiat'] profit_all_coin = stats['profit_all_coin'] profit_all_percent_mean = stats['profit_all_percent_mean'] - profit_all_percent_sum = stats['profit_all_percent_sum'] + profit_all_percent = stats['profit_all_percent'] profit_all_fiat = stats['profit_all_fiat'] trade_count = stats['trade_count'] first_trade_date = stats['first_trade_date'] @@ -514,7 +514,7 @@ class Telegram(RPCHandler): markdown_msg = ("*ROI:* Closed trades\n" f"∙ `{round_coin_value(profit_closed_coin, stake_cur)} " f"({profit_closed_percent_mean:.2f}%) " - f"({profit_closed_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" + f"({profit_closed_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" f"∙ `{round_coin_value(profit_closed_fiat, fiat_disp_cur)}`\n") else: markdown_msg = "`No closed trade` \n" @@ -523,7 +523,7 @@ class Telegram(RPCHandler): f"*ROI:* All trades\n" f"∙ `{round_coin_value(profit_all_coin, stake_cur)} " f"({profit_all_percent_mean:.2f}%) " - f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" + f"({profit_all_percent} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n" f"*Total Trade Count:* `{trade_count}`\n" f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* " @@ -598,7 +598,10 @@ class Telegram(RPCHandler): "Starting capital: " f"`{self._config['dry_run_wallet']}` {self._config['stake_currency']}.\n" ) + total_dust_balance = 0 + total_dust_currencies = 0 for curr in result['currencies']: + curr_output = '' if curr['est_stake'] > balance_dust_level: curr_output = ( f"*{curr['currency']}:*\n" @@ -607,9 +610,9 @@ class Telegram(RPCHandler): f"\t`Pending: {curr['used']:.8f}`\n" f"\t`Est. {curr['stake']}: " f"{round_coin_value(curr['est_stake'], curr['stake'], False)}`\n") - else: - curr_output = (f"*{curr['currency']}:* not showing <{balance_dust_level} " - f"{curr['stake']} amount \n") + elif curr['est_stake'] <= balance_dust_level: + total_dust_balance += curr['est_stake'] + total_dust_currencies += 1 # Handle overflowing message length if len(output + curr_output) >= MAX_TELEGRAM_MESSAGE_LENGTH: @@ -618,6 +621,14 @@ class Telegram(RPCHandler): else: output += curr_output + if total_dust_balance > 0: + output += ( + f"*{total_dust_currencies} Other " + f"{plural(total_dust_currencies, 'Currency', 'Currencies')} " + f"(< {balance_dust_level} {result['stake']}):*\n" + f"\t`Est. {result['stake']}: " + f"{round_coin_value(total_dust_balance, result['stake'], False)}`\n") + output += ("\n*Estimated Value*:\n" f"\t`{result['stake']}: {result['total']: .8f}`\n" f"\t`{result['symbol']}: " diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 36a3b0600..fa3384660 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -5,8 +5,10 @@ This module defines a base class for auto-hyperoptable strategies. import logging from abc import ABC, abstractmethod from contextlib import suppress +from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple, Union +from freqtrade.misc import deep_merge_dicts, json_load from freqtrade.optimize.hyperopt_tools import HyperoptTools @@ -333,10 +335,36 @@ class HyperStrategyMixin(object): """ Load Hyperoptable parameters """ - self._load_params(getattr(self, 'buy_params', None), 'buy', hyperopt) - self._load_params(getattr(self, 'sell_params', None), 'sell', hyperopt) + params = self.load_params_from_file() + params = params.get('params', {}) + self._ft_params_from_file = params + buy_params = deep_merge_dicts(params.get('buy', {}), getattr(self, 'buy_params', None)) + sell_params = deep_merge_dicts(params.get('sell', {}), getattr(self, 'sell_params', None)) - def _load_params(self, params: dict, space: str, hyperopt: bool = False) -> None: + self._load_params(buy_params, 'buy', hyperopt) + self._load_params(sell_params, 'sell', hyperopt) + + def load_params_from_file(self) -> Dict: + filename_str = getattr(self, '__file__', '') + if not filename_str: + return {} + filename = Path(filename_str).with_suffix('.json') + + if filename.is_file(): + logger.info(f"Loading parameters from file {filename}") + try: + params = json_load(filename.open('r')) + if params.get('strategy_name') != self.__class__.__name__: + raise OperationalException('Invalid parameter file provided.') + return params + except ValueError: + logger.warning("Invalid parameter file format.") + return {} + logger.info("Found no parameter file.") + + return {} + + def _load_params(self, params: Dict, space: str, hyperopt: bool = False) -> None: """ Set optimizable parameter values. :param params: Dictionary with new parameter values. @@ -363,7 +391,7 @@ class HyperStrategyMixin(object): else: logger.info(f'Strategy Parameter(default): {attr_name} = {attr.value}') - def get_params_dict(self): + def get_no_optimize_params(self): """ Returns list of Parameters that are not part of the current optimize job """ diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6ff499199..7158ad8e8 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -62,6 +62,7 @@ class IStrategy(ABC, HyperStrategyMixin): _populate_fun_len: int = 0 _buy_fun_len: int = 0 _sell_fun_len: int = 0 + _ft_params_from_file: Dict = {} # associated minimal roi minimal_roi: Dict @@ -303,6 +304,23 @@ class IStrategy(ABC, HyperStrategyMixin): """ return None + def custom_stake_amount(self, pair: str, current_time: datetime, current_rate: float, + proposed_stake: float, min_stake: float, max_stake: float, + **kwargs) -> float: + """ + Customize stake size for each new trade. This method is not called when edge module is + enabled. + + :param pair: Pair that's currently analyzed + :param current_time: datetime object, containing the current datetime + :param current_rate: Rate, calculated based on pricing settings in ask_strategy. + :param proposed_stake: A stake amount proposed by the bot. + :param min_stake: Minimal stake size allowed by exchange. + :param max_stake: Balance available for trading. + :return: A stake size, which is between min_stake and max_stake. + """ + return proposed_stake + def informative_pairs(self) -> ListPairsWithTimeframes: """ Define additional, informative pair/interval combinations to be cached from the exchange. diff --git a/freqtrade/templates/strategy_analysis_example.ipynb b/freqtrade/templates/strategy_analysis_example.ipynb index 0bc593e2d..99720ae6e 100644 --- a/freqtrade/templates/strategy_analysis_example.ipynb +++ b/freqtrade/templates/strategy_analysis_example.ipynb @@ -188,6 +188,52 @@ "trades.groupby(\"pair\")[\"sell_reason\"].value_counts()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plotting daily profit / equity line" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Plotting equity line (starting with 0 on day 1 and adding daily profit for each backtested day)\n", + "\n", + "from freqtrade.configuration import Configuration\n", + "from freqtrade.data.btanalysis import load_backtest_data, load_backtest_stats\n", + "import plotly.express as px\n", + "import pandas as pd\n", + "\n", + "# strategy = 'SampleStrategy'\n", + "# config = Configuration.from_files([\"user_data/config.json\"])\n", + "# backtest_dir = config[\"user_data_dir\"] / \"backtest_results\"\n", + "\n", + "stats = load_backtest_stats(backtest_dir)\n", + "strategy_stats = stats['strategy'][strategy]\n", + "\n", + "dates = []\n", + "profits = []\n", + "for date_profit in strategy_stats['daily_profit']:\n", + " dates.append(date_profit[0])\n", + " profits.append(date_profit[1])\n", + "\n", + "equity = 0\n", + "equity_daily = []\n", + "for daily_profit in profits:\n", + " equity_daily.append(equity)\n", + " equity += float(daily_profit)\n", + "\n", + "\n", + "df = pd.DataFrame({'dates': dates,'equity_daily': equity_daily})\n", + "\n", + "fig = px.line(df, x=\"dates\", y=\"equity_daily\")\n", + "fig.show()\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -329,7 +375,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.8.5" }, "mimetype": "text/x-python", "name": "python", diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index 1b2ec4550..237c1dc2c 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -70,9 +70,7 @@ class Wallets: # If not backtesting... # TODO: potentially remove the ._log workaround to determine backtest mode. if self._log: - closed_trades = Trade.get_trades_proxy(is_open=False) - tot_profit = sum( - [trade.close_profit_abs for trade in closed_trades if trade.close_profit_abs]) + tot_profit = Trade.get_total_closed_profit() else: tot_profit = LocalTrade.total_profit tot_in_trades = sum([trade.stake_amount for trade in open_trades]) @@ -131,7 +129,41 @@ class Wallets: def get_all_balances(self) -> Dict[str, Any]: return self._wallets - def _get_available_stake_amount(self, val_tied_up: float) -> float: + def get_starting_balance(self) -> float: + """ + Retrieves starting balance - based on either available capital, + or by using current balance subtracting + """ + if "available_capital" in self._config: + return self._config['available_capital'] + else: + tot_profit = Trade.get_total_closed_profit() + open_stakes = Trade.total_open_trades_stakes() + available_balance = self.get_free(self._config['stake_currency']) + return available_balance - tot_profit + open_stakes + + def get_total_stake_amount(self): + """ + Return the total currently available balance in stake currency, including tied up stake and + respecting tradable_balance_ratio. + Calculated as + ( + free amount) * tradable_balance_ratio + """ + val_tied_up = Trade.total_open_trades_stakes() + if "available_capital" in self._config: + starting_balance = self._config['available_capital'] + tot_profit = Trade.get_total_closed_profit() + available_amount = starting_balance + tot_profit + + else: + # Ensure % is used from the overall balance + # Otherwise we'd risk lowering stakes with each open trade. + # (tied up + current free) * ratio) - tied up + available_amount = ((val_tied_up + self.get_free(self._config['stake_currency'])) * + self._config['tradable_balance_ratio']) + return available_amount + + def get_available_stake_amount(self) -> float: """ Return the total currently available balance in stake currency, respecting tradable_balance_ratio. @@ -139,12 +171,8 @@ class Wallets: ( + free amount) * tradable_balance_ratio - """ - # Ensure % is used from the overall balance - # Otherwise we'd risk lowering stakes with each open trade. - # (tied up + current free) * ratio) - tied up - available_amount = ((val_tied_up + self.get_free(self._config['stake_currency'])) * - self._config['tradable_balance_ratio']) - val_tied_up - return available_amount + free = self.get_free(self._config['stake_currency']) + return min(self.get_total_stake_amount() - Trade.total_open_trades_stakes(), free) def _calculate_unlimited_stake_amount(self, available_amount: float, val_tied_up: float) -> float: @@ -193,7 +221,7 @@ class Wallets: # Ensure wallets are uptodate. self.update() val_tied_up = Trade.total_open_trades_stakes() - available_amount = self._get_available_stake_amount(val_tied_up) + available_amount = self.get_available_stake_amount() if edge: stake_amount = edge.stake_amount( @@ -209,3 +237,30 @@ class Wallets: available_amount, val_tied_up) return self._check_available_stake_amount(stake_amount, available_amount) + + def _validate_stake_amount(self, pair, stake_amount, min_stake_amount): + if not stake_amount: + logger.debug(f"Stake amount is {stake_amount}, ignoring possible trade for {pair}.") + return 0 + + max_stake_amount = self.get_available_stake_amount() + + if min_stake_amount > max_stake_amount: + if self._log: + logger.warning("Minimum stake amount > available balance.") + return 0 + if min_stake_amount is not None and stake_amount < min_stake_amount: + stake_amount = min_stake_amount + if self._log: + logger.info( + f"Stake amount for pair {pair} is too small " + f"({stake_amount} < {min_stake_amount}), adjusting to {min_stake_amount}." + ) + if stake_amount > max_stake_amount: + stake_amount = max_stake_amount + if self._log: + logger.info( + f"Stake amount for pair {pair} is too big " + f"({stake_amount} > {max_stake_amount}), adjusting to {max_stake_amount}." + ) + return stake_amount diff --git a/requirements-dev.txt b/requirements-dev.txt index c73acbe9c..e25f810cd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,7 +13,7 @@ pytest-asyncio==0.15.1 pytest-cov==2.12.1 pytest-mock==3.6.1 pytest-random-order==1.0.4 -isort==5.9.1 +isort==5.9.2 # Convert jupyter notebooks to markdown documents nbconvert==6.1.0 diff --git a/requirements-plot.txt b/requirements-plot.txt index 0563e5df2..e03fd4d66 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.0.0 +plotly==5.1.0 diff --git a/requirements.txt b/requirements.txt index a983e4797..8e26dfc6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,19 @@ -numpy==1.21.0 -pandas==1.2.5 +numpy==1.21.1 +pandas==1.3.0 -ccxt==1.52.4 +ccxt==1.53.25 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.19 -python-telegram-bot==13.6 +SQLAlchemy==1.4.21 +python-telegram-bot==13.7 arrow==1.1.1 cachetools==4.2.2 -requests==2.25.1 +requests==2.26.0 urllib3==1.26.6 wrapt==1.12.1 jsonschema==3.2.0 -TA-Lib==0.4.20 +TA-Lib==0.4.21 technical==1.3.0 tabulate==0.8.9 pycoingecko==2.2.0 @@ -31,7 +31,7 @@ python-rapidjson==1.4 sdnotify==0.3.2 # API Server -fastapi==0.65.2 +fastapi==0.66.0 uvicorn==0.14.0 pyjwt==2.1.0 aiofiles==0.7.0 @@ -39,5 +39,5 @@ aiofiles==0.7.0 # Support for colorized terminal output colorama==0.4.4 # Building config files interactively -questionary==1.9.0 +questionary==1.10.0 prompt-toolkit==3.0.19 diff --git a/setup.sh b/setup.sh index 631c31df2..a85bd3104 100755 --- a/setup.sh +++ b/setup.sh @@ -4,8 +4,12 @@ function check_installed_pip() { ${PYTHON} -m pip > /dev/null if [ $? -ne 0 ]; then - echo "pip not found (called as '${PYTHON} -m pip'). Please make sure that pip is available for ${PYTHON}." - exit 1 + echo "-----------------------------" + echo "Installing Pip for ${PYTHON}" + echo "-----------------------------" + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + ${PYTHON} get-pip.py + rm get-pip.py fi } @@ -17,35 +21,19 @@ function check_installed_python() { exit 2 fi - which python3.8 - if [ $? -eq 0 ]; then - echo "using Python 3.8" - PYTHON=python3.8 - check_installed_pip - return - fi + for v in 9 8 7 + do + PYTHON="python3.${v}" + which $PYTHON + if [ $? -eq 0 ]; then + echo "using ${PYTHON}" + check_installed_pip + return + fi + done - which python3.9 - if [ $? -eq 0 ]; then - echo "using Python 3.9" - PYTHON=python3.9 - check_installed_pip - return - fi - - which python3.7 - if [ $? -eq 0 ]; then - echo "using Python 3.7" - PYTHON=python3.7 - check_installed_pip - return - fi - - - if [ -z ${PYTHON} ]; then - echo "No usable python found. Please make sure to have python3.7 or newer installed" - exit 1 - fi + echo "No usable python found. Please make sure to have python3.7 or newer installed" + exit 1 } function updateenv() { @@ -122,6 +110,25 @@ function install_talib() { cd .. } +function install_mac_newer_python_dependencies() { + + if [ ! $(brew --prefix --installed hdf5 2>/dev/null) ] + then + echo "-------------------------" + echo "Installing hdf5" + echo "-------------------------" + brew install hdf5 + fi + + if [ ! $(brew --prefix --installed c-blosc 2>/dev/null) ] + then + echo "-------------------------" + echo "Installing c-blosc" + echo "-------------------------" + brew install c-blosc + fi +} + # Install bot MacOS function install_macos() { if [ ! -x "$(command -v brew)" ] @@ -131,14 +138,19 @@ function install_macos() { echo "-------------------------" /usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" fi + #Gets number after decimal in python version + version=$(egrep -o 3.\[0-9\]+ <<< $PYTHON | sed 's/3.//g') + + if [[ $version -ge 9 ]]; then #Checks if python version >= 3.9 + install_mac_newer_python_dependencies + fi install_talib - test_and_fix_python_on_mac } # Install bot Debian_ubuntu function install_debian() { sudo apt-get update - sudo apt-get install -y build-essential autoconf libtool pkg-config make wget git libpython3-dev + sudo apt-get install -y build-essential autoconf libtool pkg-config make wget git $(echo lib${PYTHON}-dev ${PYTHON}-venv) install_talib } @@ -189,19 +201,6 @@ function reset() { updateenv } -function test_and_fix_python_on_mac() { - - if ! [ -x "$(command -v python3.6)" ] - then - echo "-------------------------" - echo "Fixing Python" - echo "-------------------------" - echo "Python 3.6 is not linked in your system. Fixing it..." - brew link --overwrite python - echo - fi -} - function config() { echo "-------------------------" @@ -240,12 +239,12 @@ function install() { } function plot() { -echo " ------------------------------------------ -Installing dependencies for Plotting scripts ------------------------------------------ -" -${PYTHON} -m pip install plotly --upgrade + echo " + ----------------------------------------- + Installing dependencies for Plotting scripts + ----------------------------------------- + " + ${PYTHON} -m pip install plotly --upgrade } function help() { diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 47f298ad7..c0268038a 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -13,7 +13,7 @@ from freqtrade.commands import (start_convert_data, start_create_userdir, start_ start_list_data, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, start_list_timeframes, start_new_hyperopt, start_new_strategy, start_show_trades, - start_test_pairlist, start_trading) + start_test_pairlist, start_trading, start_webserver) from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration @@ -26,7 +26,7 @@ from tests.conftest_trades import MOCK_TRADE_COUNT def test_setup_utils_configuration(): args = [ - 'list-exchanges', '--config', 'config_bittrex.json.example', + 'list-exchanges', '--config', 'config_examples/config_bittrex.example.json', ] config = setup_utils_configuration(get_args(args), RunMode.OTHER) @@ -45,7 +45,7 @@ def test_start_trading_fail(mocker, caplog): exitmock = mocker.patch("freqtrade.worker.Worker.exit", MagicMock()) args = [ 'trade', - '-c', 'config_bittrex.json.example' + '-c', 'config_examples/config_bittrex.example.json' ] start_trading(get_args(args)) assert exitmock.call_count == 1 @@ -58,6 +58,18 @@ def test_start_trading_fail(mocker, caplog): assert log_has('Fatal exception!', caplog) +def test_start_webserver(mocker, caplog): + + api_server_mock = mocker.patch("freqtrade.rpc.api_server.ApiServer", ) + + args = [ + 'webserver', + '-c', 'config_examples/config_bittrex.example.json' + ] + start_webserver(get_args(args)) + assert api_server_mock.call_count == 1 + + def test_list_exchanges(capsys): args = [ @@ -127,10 +139,10 @@ def test_list_timeframes(mocker, capsys): match=r"This command requires a configured exchange.*"): start_list_timeframes(pargs) - # Test with --config config_bittrex.json.example + # Test with --config config_examples/config_bittrex.example.json args = [ "list-timeframes", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', ] start_list_timeframes(get_args(args)) captured = capsys.readouterr() @@ -174,7 +186,7 @@ def test_list_timeframes(mocker, capsys): # Test with --one-column args = [ "list-timeframes", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--one-column", ] start_list_timeframes(get_args(args)) @@ -214,10 +226,10 @@ def test_list_markets(mocker, markets, capsys): match=r"This command requires a configured exchange.*"): start_list_markets(pargs, False) - # Test with --config config_bittrex.json.example + # Test with --config config_examples/config_bittrex.example.json args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--print-list", ] start_list_markets(get_args(args), False) @@ -244,7 +256,7 @@ def test_list_markets(mocker, markets, capsys): # Test with --all: all markets args = [ "list-markets", "--all", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--print-list", ] start_list_markets(get_args(args), False) @@ -257,7 +269,7 @@ def test_list_markets(mocker, markets, capsys): # Test list-pairs subcommand: active pairs args = [ "list-pairs", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--print-list", ] start_list_markets(get_args(args), True) @@ -269,7 +281,7 @@ def test_list_markets(mocker, markets, capsys): # Test list-pairs subcommand with --all: all pairs args = [ "list-pairs", "--all", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--print-list", ] start_list_markets(get_args(args), True) @@ -282,7 +294,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=ETH, LTC args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--base", "ETH", "LTC", "--print-list", ] @@ -295,7 +307,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=LTC args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--base", "LTC", "--print-list", ] @@ -308,7 +320,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, quote=USDT, USD args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--quote", "USDT", "USD", "--print-list", ] @@ -321,7 +333,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, quote=USDT args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--quote", "USDT", "--print-list", ] @@ -334,7 +346,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=LTC, quote=USDT args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--base", "LTC", "--quote", "USDT", "--print-list", ] @@ -347,7 +359,7 @@ def test_list_markets(mocker, markets, capsys): # active pairs, base=LTC, quote=USDT args = [ "list-pairs", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--base", "LTC", "--quote", "USD", "--print-list", ] @@ -360,7 +372,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=LTC, quote=USDT, NONEXISTENT args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--base", "LTC", "--quote", "USDT", "NONEXISTENT", "--print-list", ] @@ -373,7 +385,7 @@ def test_list_markets(mocker, markets, capsys): # active markets, base=LTC, quote=NONEXISTENT args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--base", "LTC", "--quote", "NONEXISTENT", "--print-list", ] @@ -386,7 +398,7 @@ def test_list_markets(mocker, markets, capsys): # Test tabular output args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', ] start_list_markets(get_args(args), False) captured = capsys.readouterr() @@ -396,7 +408,7 @@ def test_list_markets(mocker, markets, capsys): # Test tabular output, no markets found args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--base", "LTC", "--quote", "NONEXISTENT", ] start_list_markets(get_args(args), False) @@ -408,7 +420,7 @@ def test_list_markets(mocker, markets, capsys): # Test --print-json args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--print-json" ] start_list_markets(get_args(args), False) @@ -420,7 +432,7 @@ def test_list_markets(mocker, markets, capsys): # Test --print-csv args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--print-csv" ] start_list_markets(get_args(args), False) @@ -432,7 +444,7 @@ def test_list_markets(mocker, markets, capsys): # Test --one-column args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--one-column" ] start_list_markets(get_args(args), False) @@ -444,7 +456,7 @@ def test_list_markets(mocker, markets, capsys): # Test --one-column args = [ "list-markets", - '--config', 'config_bittrex.json.example', + '--config', 'config_examples/config_bittrex.example.json', "--one-column" ] with pytest.raises(OperationalException, match=r"Cannot get markets.*"): @@ -887,7 +899,7 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): patched_configuration_load_config_file(mocker, default_conf) args = [ 'test-pairlist', - '-c', 'config_bittrex.json.example' + '-c', 'config_examples/config_bittrex.example.json' ] start_test_pairlist(get_args(args)) @@ -901,7 +913,7 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): args = [ 'test-pairlist', - '-c', 'config_bittrex.json.example', + '-c', 'config_examples/config_bittrex.example.json', '--one-column', ] start_test_pairlist(get_args(args)) @@ -910,7 +922,7 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): args = [ 'test-pairlist', - '-c', 'config_bittrex.json.example', + '-c', 'config_examples/config_bittrex.example.json', '--print-json', ] start_test_pairlist(get_args(args)) @@ -1168,6 +1180,7 @@ def test_hyperopt_show(mocker, capsys, saved_hyperopt_results): 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', MagicMock(return_value=saved_hyperopt_results) ) + mocker.patch('freqtrade.commands.hyperopt_commands.show_backtest_result') args = [ "hyperopt-show", diff --git a/tests/conftest.py b/tests/conftest.py index 58af2fe90..139d989a9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -324,6 +324,7 @@ def get_default_conf(testdatadir): "verbosity": 3, "strategy_path": str(Path(__file__).parent / "strategy" / "strats"), "strategy": "DefaultStrategy", + "disableparamexport": True, "internals": {}, "export": "none", } @@ -1761,7 +1762,7 @@ def rpc_balance(): 'total': 0.1, 'free': 0.01, 'used': 0.0 - }, + }, 'EUR': { 'total': 10.0, 'free': 10.0, @@ -1953,12 +1954,13 @@ def saved_hyperopt_results(): 'params_dict': { 'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal', 'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper', 'roi_t1': 1190, 'roi_t2': 541, 'roi_t3': 408, 'roi_p1': 0.026035863879169705, 'roi_p2': 0.12508730043628782, 'roi_p3': 0.27766427921605896, 'stoploss': -0.2562930402099556}, # noqa: E501 'params_details': {'buy': {'mfi-value': 15, 'fastd-value': 20, 'adx-value': 25, 'rsi-value': 28, 'mfi-enabled': False, 'fastd-enabled': True, 'adx-enabled': True, 'rsi-enabled': True, 'trigger': 'macd_cross_signal'}, 'sell': {'sell-mfi-value': 88, 'sell-fastd-value': 97, 'sell-adx-value': 51, 'sell-rsi-value': 67, 'sell-mfi-enabled': False, 'sell-fastd-enabled': False, 'sell-adx-enabled': True, 'sell-rsi-enabled': True, 'sell-trigger': 'sell-bb_upper'}, 'roi': {0: 0.4287874435315165, 408: 0.15112316431545753, 949: 0.026035863879169705, 2139: 0}, 'stoploss': {'stoploss': -0.2562930402099556}}, # noqa: E501 - 'results_metrics': {'total_trades': 2, 'wins': 0, 'draws': 0, 'losses': 2, 'profit_mean': -0.01254995, 'profit_median': -0.012222, 'profit_total': -0.00125625, 'profit_total_abs': -2.50999, 'holding_avg': timedelta(minutes=3930.0)}, # noqa: E501 + 'results_metrics': {'total_trades': 2, 'wins': 0, 'draws': 0, 'losses': 2, 'profit_mean': -0.01254995, 'profit_median': -0.012222, 'profit_total': -0.00125625, 'profit_total_abs': -2.50999, 'holding_avg': timedelta(minutes=3930.0), 'stake_currency': 'BTC', 'strategy_name': 'SampleStrategy'}, # noqa: E501 'results_explanation': ' 2 trades. Avg profit -1.25%. Total profit -0.00125625 BTC ( -2.51Σ%). Avg duration 3930.0 min.', # noqa: E501 'total_profit': -0.00125625, 'current_epoch': 1, 'is_initial_point': True, - 'is_best': True + 'is_best': True, + }, { 'loss': 20.0, 'params_dict': { diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 524dc873c..02adf01c4 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1783,14 +1783,14 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'ask': ask, 'last': last, 'bid': bid}) - assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert exchange.get_rate('ETH/BTC', refresh=True, side="buy") == expected assert not log_has("Using cached buy rate for ETH/BTC.", caplog) - assert exchange.get_buy_rate('ETH/BTC', False) == expected + assert exchange.get_rate('ETH/BTC', refresh=False, side="buy") == expected assert log_has("Using cached buy rate for ETH/BTC.", caplog) # Running a 2nd time with Refresh on! caplog.clear() - assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert exchange.get_rate('ETH/BTC', refresh=True, side="buy") == expected assert not log_has("Using cached buy rate for ETH/BTC.", caplog) @@ -1825,12 +1825,12 @@ def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, # Test regular mode exchange = get_patched_exchange(mocker, default_conf) - rate = exchange.get_sell_rate(pair, True) + rate = exchange.get_rate(pair, refresh=True, side="sell") assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) assert rate == expected # Use caching - rate = exchange.get_sell_rate(pair, False) + rate = exchange.get_rate(pair, refresh=False, side="sell") assert rate == expected assert log_has("Using cached sell rate for ETH/BTC.", caplog) @@ -1848,11 +1848,11 @@ def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, o pair = "ETH/BTC" mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) exchange = get_patched_exchange(mocker, default_conf) - rate = exchange.get_sell_rate(pair, True) + rate = exchange.get_rate(pair, refresh=True, side="sell") assert not log_has("Using cached sell rate for ETH/BTC.", caplog) assert isinstance(rate, float) assert rate == expected - rate = exchange.get_sell_rate(pair, False) + rate = exchange.get_rate(pair, refresh=False, side="sell") assert rate == expected assert log_has("Using cached sell rate for ETH/BTC.", caplog) @@ -1868,7 +1868,7 @@ def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog): return_value={'bids': [[]], 'asks': [[]]}) exchange = get_patched_exchange(mocker, default_conf) with pytest.raises(PricingError): - exchange.get_sell_rate(pair, True) + exchange.get_rate(pair, refresh=True, side="sell") assert log_has_re(r"Sell Price at location 1 from orderbook could not be determined\..*", caplog) @@ -1881,18 +1881,18 @@ def test_get_sell_rate_exception(default_conf, mocker, caplog): return_value={'ask': None, 'bid': 0.12, 'last': None}) exchange = get_patched_exchange(mocker, default_conf) with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): - exchange.get_sell_rate(pair, True) + exchange.get_rate(pair, refresh=True, side="sell") exchange._config['ask_strategy']['price_side'] = 'bid' - assert exchange.get_sell_rate(pair, True) == 0.12 + assert exchange.get_rate(pair, refresh=True, side="sell") == 0.12 # Reverse sides mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'ask': 0.13, 'bid': None, 'last': None}) with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): - exchange.get_sell_rate(pair, True) + exchange.get_rate(pair, refresh=True, side="sell") exchange._config['ask_strategy']['price_side'] = 'ask' - assert exchange.get_sell_rate(pair, True) == 0.13 + assert exchange.get_rate(pair, refresh=True, side="sell") == 0.13 def make_fetch_ohlcv_mock(data): @@ -2203,7 +2203,7 @@ def test_cancel_order_dry_run(default_conf, mocker, exchange_name): ({'status': 'canceled', 'filled': 10.0}, False), ({'status': 'unknown', 'filled': 10.0}, False), ({'result': 'testest123'}, False), - ]) +]) def test_check_order_canceled_empty(mocker, default_conf, exchange_name, order, result): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) assert exchange.check_order_canceled_empty(order) == result diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 102d62273..655464344 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -346,6 +346,20 @@ def test_data_to_dataframe_bt(default_conf, mocker, testdatadir) -> None: assert processed['UNITTEST/BTC'].equals(processed2['UNITTEST/BTC']) +def test_backtest_abort(default_conf, mocker, testdatadir) -> None: + patch_exchange(mocker) + backtesting = Backtesting(default_conf) + backtesting.check_abort() + + backtesting.abort = True + + with pytest.raises(DependencyException, match="Stop requested"): + backtesting.check_abort() + # abort flag resets + assert backtesting.abort is False + assert backtesting.progress.progress == 0 + + def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: def get_timerange(input1): return Arrow(2017, 11, 14, 21, 17), Arrow(2017, 11, 14, 22, 59) @@ -497,6 +511,17 @@ def test_backtest__enter_trade(default_conf, fee, mocker) -> None: trade = backtesting._enter_trade(pair, row=row) assert trade is not None + backtesting.strategy.custom_stake_amount = lambda **kwargs: 123.5 + trade = backtesting._enter_trade(pair, row=row) + assert trade + assert trade.stake_amount == 123.5 + + # In case of error - use proposed stake + backtesting.strategy.custom_stake_amount = lambda **kwargs: 20 / 0 + trade = backtesting._enter_trade(pair, row=row) + assert trade + assert trade.stake_amount == 495 + # Stake-amount too high! mocker.patch("freqtrade.exchange.Exchange.get_min_pair_stake_amount", return_value=600.0) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 10e99395d..14fea573f 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,9 +1,6 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import logging -import re from datetime import datetime from pathlib import Path -from typing import Dict, List from unittest.mock import ANY, MagicMock import pandas as pd @@ -28,12 +25,6 @@ from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, from .hyperopts.default_hyperopt import DefaultHyperOpt -# Functions for recurrent object patching -def create_results() -> List[Dict]: - - return [{'loss': 1, 'result': 'foo', 'params': {}, 'is_best': True}] - - def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) @@ -303,52 +294,6 @@ def test_no_log_if_loss_does_not_improve(hyperopt, caplog) -> None: assert caplog.record_tuples == [] -def test_save_results_saves_epochs(mocker, hyperopt, tmpdir, caplog) -> None: - # Test writing to temp dir and reading again - epochs = create_results() - hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt') - - caplog.set_level(logging.DEBUG) - - for epoch in epochs: - hyperopt._save_result(epoch) - assert log_has(f"1 epoch saved to '{hyperopt.results_file}'.", caplog) - - hyperopt._save_result(epochs[0]) - assert log_has(f"2 epochs saved to '{hyperopt.results_file}'.", caplog) - - hyperopt_epochs = HyperoptTools.load_previous_results(hyperopt.results_file) - assert len(hyperopt_epochs) == 2 - - -def test_load_previous_results(testdatadir, caplog) -> None: - - results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' - - hyperopt_epochs = HyperoptTools.load_previous_results(results_file) - - assert len(hyperopt_epochs) == 5 - assert log_has_re(r"Reading pickled epochs from .*", caplog) - - caplog.clear() - - # Modern version - results_file = testdatadir / 'strategy_SampleStrategy.fthypt' - - hyperopt_epochs = HyperoptTools.load_previous_results(results_file) - - assert len(hyperopt_epochs) == 5 - assert log_has_re(r"Reading epochs from .*", caplog) - - -def test_load_previous_results2(mocker, testdatadir, caplog) -> None: - mocker.patch('freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results_pickle', - return_value=[{'asdf': '222'}]) - results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' - with pytest.raises(OperationalException, match=r"The file .* incompatible.*"): - HyperoptTools.load_previous_results(results_file) - - def test_roi_table_generation(hyperopt) -> None: params = { 'roi_t1': 5, @@ -362,6 +307,18 @@ def test_roi_table_generation(hyperopt) -> None: assert hyperopt.custom_hyperopt.generate_roi_table(params) == {0: 6, 15: 3, 25: 1, 30: 0} +def test_params_no_optimize_details(hyperopt) -> None: + hyperopt.config['spaces'] = ['buy'] + res = hyperopt._get_no_optimize_details() + assert isinstance(res, dict) + assert "trailing" in res + assert res["trailing"]['trailing_stop'] is False + assert "roi" in res + assert res['roi']['0'] == 0.04 + assert "stoploss" in res + assert res['stoploss']['stoploss'] == -0.1 + + def test_start_calls_optimizer(mocker, hyperopt_conf, capsys) -> None: dumper = mocker.patch('freqtrade.optimize.hyperopt.dump') dumper2 = mocker.patch('freqtrade.optimize.hyperopt.Hyperopt._save_result') @@ -467,40 +424,6 @@ def test_hyperopt_format_results(hyperopt): assert '0:50:00 min' in result -@pytest.mark.parametrize("spaces, expected_results", [ - (['buy'], - {'buy': True, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False}), - (['sell'], - {'buy': False, 'sell': True, 'roi': False, 'stoploss': False, 'trailing': False}), - (['roi'], - {'buy': False, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), - (['stoploss'], - {'buy': False, 'sell': False, 'roi': False, 'stoploss': True, 'trailing': False}), - (['trailing'], - {'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': True}), - (['buy', 'sell', 'roi', 'stoploss'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), - (['buy', 'sell', 'roi', 'stoploss', 'trailing'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), - (['buy', 'roi'], - {'buy': True, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), - (['all'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), - (['default'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), - (['default', 'trailing'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), - (['all', 'buy'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), - (['default', 'buy'], - {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), -]) -def test_has_space(hyperopt_conf, spaces, expected_results): - for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: - hyperopt_conf.update({'spaces': spaces}) - assert HyperoptTools.has_space(hyperopt_conf, s) == expected_results[s] - - def test_populate_indicators(hyperopt, testdatadir) -> None: data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) dataframes = hyperopt.backtesting.strategy.ohlcvdata_to_dataframe(data) @@ -686,6 +609,8 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: def test_clean_hyperopt(mocker, hyperopt_conf, caplog): patch_exchange(mocker) + mocker.patch("freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file", + MagicMock(return_value={})) mocker.patch("freqtrade.optimize.hyperopt.Path.is_file", MagicMock(return_value=True)) unlinkmock = mocker.patch("freqtrade.optimize.hyperopt.Path.unlink", MagicMock()) h = Hyperopt(hyperopt_conf) @@ -1068,42 +993,6 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No hyperopt.start() -def test_show_epoch_details(capsys): - test_result = { - 'params_details': { - 'trailing': { - 'trailing_stop': True, - 'trailing_stop_positive': 0.02, - 'trailing_stop_positive_offset': 0.04, - 'trailing_only_offset_is_reached': True - }, - 'roi': { - 0: 0.18, - 90: 0.14, - 225: 0.05, - 430: 0}, - }, - 'results_explanation': 'foo result', - 'is_initial_point': False, - 'total_profit': 0, - 'current_epoch': 2, # This starts from 1 (in a human-friendly manner) - 'is_best': True - } - - HyperoptTools.show_epoch_details(test_result, 5, False, no_header=True) - captured = capsys.readouterr() - assert '# Trailing stop:' in captured.out - # re.match(r"Pairs for .*", captured.out) - assert re.search(r'^\s+trailing_stop = True$', captured.out, re.MULTILINE) - assert re.search(r'^\s+trailing_stop_positive = 0.02$', captured.out, re.MULTILINE) - assert re.search(r'^\s+trailing_stop_positive_offset = 0.04$', captured.out, re.MULTILINE) - assert re.search(r'^\s+trailing_only_offset_is_reached = True$', captured.out, re.MULTILINE) - - assert '# ROI table:' in captured.out - assert re.search(r'^\s+minimal_roi = \{$', captured.out, re.MULTILINE) - assert re.search(r'^\s+\"90\"\:\s0.14,\s*$', captured.out, re.MULTILINE) - - def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: patch_exchange(mocker) mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) @@ -1143,17 +1032,3 @@ def test_SKDecimal(): assert space.transform([2.0]) == [200] assert space.transform([1.0]) == [100] assert space.transform([1.5, 1.6]) == [150, 160] - - -def test___pprint(): - params = {'buy_std': 1.2, 'buy_rsi': 31, 'buy_enable': True, 'buy_what': 'asdf'} - non_params = {'buy_notoptimied': 55} - - x = HyperoptTools._pprint(params, non_params) - assert x == """{ - "buy_std": 1.2, - "buy_rsi": 31, - "buy_enable": True, - "buy_what": "asdf", - "buy_notoptimied": 55, # value loaded from strategy -}""" diff --git a/tests/optimize/test_hyperopt_tools.py b/tests/optimize/test_hyperopt_tools.py new file mode 100644 index 000000000..44b4a7a03 --- /dev/null +++ b/tests/optimize/test_hyperopt_tools.py @@ -0,0 +1,317 @@ +import logging +import re +from pathlib import Path +from typing import Dict, List + +import numpy as np +import pytest +import rapidjson + +from freqtrade.constants import FTHYPT_FILEVERSION +from freqtrade.exceptions import OperationalException +from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer +from tests.conftest import log_has, log_has_re + + +# Functions for recurrent object patching +def create_results() -> List[Dict]: + + return [{'loss': 1, 'result': 'foo', 'params': {}, 'is_best': True}] + + +def test_save_results_saves_epochs(hyperopt, tmpdir, caplog) -> None: + # Test writing to temp dir and reading again + epochs = create_results() + hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt') + + caplog.set_level(logging.DEBUG) + + for epoch in epochs: + hyperopt._save_result(epoch) + assert log_has(f"1 epoch saved to '{hyperopt.results_file}'.", caplog) + + hyperopt._save_result(epochs[0]) + assert log_has(f"2 epochs saved to '{hyperopt.results_file}'.", caplog) + + hyperopt_epochs = HyperoptTools.load_previous_results(hyperopt.results_file) + assert len(hyperopt_epochs) == 2 + + +def test_load_previous_results(testdatadir, caplog) -> None: + + results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' + + hyperopt_epochs = HyperoptTools.load_previous_results(results_file) + + assert len(hyperopt_epochs) == 5 + assert log_has_re(r"Reading pickled epochs from .*", caplog) + + caplog.clear() + + # Modern version + results_file = testdatadir / 'strategy_SampleStrategy.fthypt' + + hyperopt_epochs = HyperoptTools.load_previous_results(results_file) + + assert len(hyperopt_epochs) == 5 + assert log_has_re(r"Reading epochs from .*", caplog) + + +def test_load_previous_results2(mocker, testdatadir, caplog) -> None: + mocker.patch('freqtrade.optimize.hyperopt_tools.HyperoptTools._read_results_pickle', + return_value=[{'asdf': '222'}]) + results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle' + with pytest.raises(OperationalException, match=r"The file .* incompatible.*"): + HyperoptTools.load_previous_results(results_file) + + +@pytest.mark.parametrize("spaces, expected_results", [ + (['buy'], + {'buy': True, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': False}), + (['sell'], + {'buy': False, 'sell': True, 'roi': False, 'stoploss': False, 'trailing': False}), + (['roi'], + {'buy': False, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), + (['stoploss'], + {'buy': False, 'sell': False, 'roi': False, 'stoploss': True, 'trailing': False}), + (['trailing'], + {'buy': False, 'sell': False, 'roi': False, 'stoploss': False, 'trailing': True}), + (['buy', 'sell', 'roi', 'stoploss'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), + (['buy', 'sell', 'roi', 'stoploss', 'trailing'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['buy', 'roi'], + {'buy': True, 'sell': False, 'roi': True, 'stoploss': False, 'trailing': False}), + (['all'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['default'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), + (['default', 'trailing'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['all', 'buy'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': True}), + (['default', 'buy'], + {'buy': True, 'sell': True, 'roi': True, 'stoploss': True, 'trailing': False}), +]) +def test_has_space(hyperopt_conf, spaces, expected_results): + for s in ['buy', 'sell', 'roi', 'stoploss', 'trailing']: + hyperopt_conf.update({'spaces': spaces}) + assert HyperoptTools.has_space(hyperopt_conf, s) == expected_results[s] + + +def test_show_epoch_details(capsys): + test_result = { + 'params_details': { + 'trailing': { + 'trailing_stop': True, + 'trailing_stop_positive': 0.02, + 'trailing_stop_positive_offset': 0.04, + 'trailing_only_offset_is_reached': True + }, + 'roi': { + 0: 0.18, + 90: 0.14, + 225: 0.05, + 430: 0}, + }, + 'results_explanation': 'foo result', + 'is_initial_point': False, + 'total_profit': 0, + 'current_epoch': 2, # This starts from 1 (in a human-friendly manner) + 'is_best': True + } + + HyperoptTools.show_epoch_details(test_result, 5, False, no_header=True) + captured = capsys.readouterr() + assert '# Trailing stop:' in captured.out + # re.match(r"Pairs for .*", captured.out) + assert re.search(r'^\s+trailing_stop = True$', captured.out, re.MULTILINE) + assert re.search(r'^\s+trailing_stop_positive = 0.02$', captured.out, re.MULTILINE) + assert re.search(r'^\s+trailing_stop_positive_offset = 0.04$', captured.out, re.MULTILINE) + assert re.search(r'^\s+trailing_only_offset_is_reached = True$', captured.out, re.MULTILINE) + + assert '# ROI table:' in captured.out + assert re.search(r'^\s+minimal_roi = \{$', captured.out, re.MULTILINE) + assert re.search(r'^\s+\"90\"\:\s0.14,\s*$', captured.out, re.MULTILINE) + + +def test__pprint_dict(): + params = {'buy_std': 1.2, 'buy_rsi': 31, 'buy_enable': True, 'buy_what': 'asdf'} + non_params = {'buy_notoptimied': 55} + + x = HyperoptTools._pprint_dict(params, non_params) + assert x == """{ + "buy_std": 1.2, + "buy_rsi": 31, + "buy_enable": True, + "buy_what": "asdf", + "buy_notoptimied": 55, # value loaded from strategy +}""" + + +def test_get_strategy_filename(default_conf): + + x = HyperoptTools.get_strategy_filename(default_conf, 'DefaultStrategy') + assert isinstance(x, Path) + assert x == Path(__file__).parents[1] / 'strategy/strats/default_strategy.py' + + x = HyperoptTools.get_strategy_filename(default_conf, 'NonExistingStrategy') + assert x is None + + +def test_export_params(tmpdir): + + filename = Path(tmpdir) / "DefaultStrategy.json" + assert not filename.is_file() + params = { + "params_details": { + "buy": { + "buy_rsi": 30 + }, + "sell": { + "sell_rsi": 70 + }, + "roi": { + "0": 0.528, + "346": 0.08499, + "507": 0.049, + "1595": 0 + } + }, + "params_not_optimized": { + "stoploss": -0.05, + "trailing": { + "trailing_stop": False, + "trailing_stop_positive": 0.05, + "trailing_stop_positive_offset": 0.1, + "trailing_only_offset_is_reached": True + }, + } + + } + HyperoptTools.export_params(params, "DefaultStrategy", filename) + + assert filename.is_file() + + content = rapidjson.load(filename.open('r')) + assert content['strategy_name'] == 'DefaultStrategy' + assert 'params' in content + assert "buy" in content["params"] + assert "sell" in content["params"] + assert "roi" in content["params"] + assert "stoploss" in content["params"] + assert "trailing" in content["params"] + + +def test_try_export_params(default_conf, tmpdir, caplog, mocker): + default_conf['disableparamexport'] = False + export_mock = mocker.patch("freqtrade.optimize.hyperopt_tools.HyperoptTools.export_params") + + filename = Path(tmpdir) / "DefaultStrategy.json" + assert not filename.is_file() + params = { + "params_details": { + "buy": { + "buy_rsi": 30 + }, + "sell": { + "sell_rsi": 70 + }, + "roi": { + "0": 0.528, + "346": 0.08499, + "507": 0.049, + "1595": 0 + } + }, + "params_not_optimized": { + "stoploss": -0.05, + "trailing": { + "trailing_stop": False, + "trailing_stop_positive": 0.05, + "trailing_stop_positive_offset": 0.1, + "trailing_only_offset_is_reached": True + }, + }, + FTHYPT_FILEVERSION: 2, + + } + HyperoptTools.try_export_params(default_conf, "DefaultStrategy22", params) + + assert log_has("Strategy not found, not exporting parameter file.", caplog) + assert export_mock.call_count == 0 + caplog.clear() + + HyperoptTools.try_export_params(default_conf, "DefaultStrategy", params) + + assert export_mock.call_count == 1 + assert export_mock.call_args_list[0][0][1] == 'DefaultStrategy' + assert export_mock.call_args_list[0][0][2].name == 'default_strategy.json' + + +def test_params_print(capsys): + + params = { + "buy": { + "buy_rsi": 30 + }, + "sell": { + "sell_rsi": 70 + }, + } + non_optimized = { + "buy": { + "buy_adx": 44 + }, + "sell": { + "sell_adx": 65 + }, + "stoploss": { + "stoploss": -0.05, + }, + "roi": { + "0": 0.05, + "20": 0.01, + }, + "trailing": { + "trailing_stop": False, + "trailing_stop_positive": 0.05, + "trailing_stop_positive_offset": 0.1, + "trailing_only_offset_is_reached": True + }, + + } + HyperoptTools._params_pretty_print(params, 'buy', 'No header', non_optimized) + + captured = capsys.readouterr() + assert re.search("# No header", captured.out) + assert re.search('"buy_rsi": 30,\n', captured.out) + assert re.search('"buy_adx": 44, # value loaded.*\n', captured.out) + assert not re.search("sell", captured.out) + + HyperoptTools._params_pretty_print(params, 'sell', 'Sell Header', non_optimized) + captured = capsys.readouterr() + assert re.search("# Sell Header", captured.out) + assert re.search('"sell_rsi": 70,\n', captured.out) + assert re.search('"sell_adx": 65, # value loaded.*\n', captured.out) + + HyperoptTools._params_pretty_print(params, 'roi', 'ROI Table:', non_optimized) + captured = capsys.readouterr() + assert re.search("# ROI Table: # value loaded.*\n", captured.out) + assert re.search('minimal_roi = {\n', captured.out) + assert re.search('"20": 0.01\n', captured.out) + + HyperoptTools._params_pretty_print(params, 'trailing', 'Trailing stop:', non_optimized) + captured = capsys.readouterr() + assert re.search("# Trailing stop:", captured.out) + assert re.search('trailing_stop = False # value loaded.*\n', captured.out) + assert re.search('trailing_stop_positive = 0.05 # value loaded.*\n', captured.out) + assert re.search('trailing_stop_positive_offset = 0.1 # value loaded.*\n', captured.out) + assert re.search('trailing_only_offset_is_reached = True # value loaded.*\n', captured.out) + + +def test_hyperopt_serializer(): + + assert isinstance(hyperopt_serializer(np.int_(5)), int) + assert isinstance(hyperopt_serializer(np.bool_(True)), bool) + assert isinstance(hyperopt_serializer(np.bool_(False)), bool) diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 3dee47599..510e9a0ae 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -79,7 +79,8 @@ def whitelist_conf_agefilter(default_conf): }, { "method": "AgeFilter", - "min_days_listed": 2 + "min_days_listed": 2, + "max_days_listed": 100 } ] return default_conf @@ -302,7 +303,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # No pair for ETH, all handlers ([{"method": "StaticPairList"}, {"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "AgeFilter", "min_days_listed": 2}, + {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": None}, {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.03}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, @@ -310,12 +311,24 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "ETH", []), # AgeFilter and VolumePairList (require 2 days only, all should pass age test) ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "AgeFilter", "min_days_listed": 2}], + {"method": "AgeFilter", "min_days_listed": 2, "max_days_listed": 100}], "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'XRP/BTC', 'HOT/BTC']), # AgeFilter and VolumePairList (require 10 days, all should fail age test) ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, - {"method": "AgeFilter", "min_days_listed": 10}], + {"method": "AgeFilter", "min_days_listed": 10, "max_days_listed": None}], "BTC", []), + # AgeFilter and VolumePairList (all pair listed > 2, all should fail age test) + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 1, "max_days_listed": 2}], + "BTC", []), + # AgeFilter and VolumePairList LTC/BTC has 6 candles - removes all + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 5}], + "BTC", []), + # AgeFilter and VolumePairList LTC/BTC has 6 candles - passes + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, + {"method": "AgeFilter", "min_days_listed": 4, "max_days_listed": 10}], + "BTC", ["LTC/BTC"]), # Precisionfilter and quote volume ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, {"method": "PrecisionFilter"}], @@ -417,7 +430,19 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): ([{"method": "StaticPairList"}, {"method": "VolatilityFilter", "lookback_days": 3, "min_volatility": 0.002, "max_volatility": 0.004, "refresh_period": 1440}], - "BTC", ['ETH/BTC', 'TKN/BTC']) + "BTC", ['ETH/BTC', 'TKN/BTC']), + # VolumePairList with no offset = unchanged pairlist + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 0}], + "USDT", ['ETH/USDT', 'NANO/USDT', 'ADAHALF/USDT', 'ADADOUBLE/USDT']), + # VolumePairList with offset = 2 + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 2}], + "USDT", ['ADAHALF/USDT', 'ADADOUBLE/USDT']), + # VolumePairList with higher offset, than total pairlist + ([{"method": "VolumePairList", "number_assets": 20, "sort_key": "quoteVolume"}, + {"method": "OffsetFilter", "offset": 100}], + "USDT", []) ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history, pairlists, base_currency, @@ -431,7 +456,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ohlcv_data = { ('ETH/BTC', '1d'): ohlcv_history, ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history.append(ohlcv_history), ('XRP/BTC', '1d'): ohlcv_history, ('HOT/BTC', '1d'): ohlcv_history_high_vola, } @@ -480,9 +505,13 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t for pairlist in pairlists: if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \ - len(ohlcv_history) <= pairlist['min_days_listed']: + len(ohlcv_history) < pairlist['min_days_listed']: assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' r'.* day.*', caplog) + if pairlist['method'] == 'AgeFilter' and pairlist['max_days_listed'] and \ + len(ohlcv_history) > pairlist['max_days_listed']: + assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' + r'.* day.* or more than .* day', caplog) if pairlist['method'] == 'PrecisionFilter' and whitelist_result: assert log_has_re(r'^Removed .* from whitelist, because stop price .* ' r'would be <= stop limit.*', caplog) @@ -507,6 +536,105 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t assert log_has_re(r'^Removed .* from whitelist, because volatility.*$', caplog) +@pytest.mark.parametrize("pairlists,base_currency,volumefilter_result", [ + # default refresh of 1800 to small for daily candle lookback + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_days": 1}], + "BTC", "default_refresh_too_short"), # OperationalException expected + # ambigous configuration with lookback days and period + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_days": 1, "lookback_period": 1}], + "BTC", "lookback_days_and_period"), # OperationalException expected + # negative lookback period + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1d", "lookback_period": -1}], + "BTC", "lookback_period_negative"), # OperationalException expected + # lookback range exceedes exchange limit + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1m", "lookback_period": 2000, "refresh_period": 3600}], + "BTC", 'lookback_exceeds_exchange_request_size'), # OperationalException expected + # expecing pairs as given + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1d", "lookback_period": 1, "refresh_period": 86400}], + "BTC", ['HOT/BTC', 'LTC/BTC', 'ETH/BTC', 'TKN/BTC', 'XRP/BTC']), + # expecting pairs from default tickers, because 1h candles are not available + ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume", + "lookback_timeframe": "1h", "lookback_period": 2, "refresh_period": 3600}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'LTC/BTC', 'HOT/BTC', 'FUEL/BTC']), +]) +def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history, + pairlists, base_currency, volumefilter_result, caplog) -> None: + whitelist_conf['pairlists'] = pairlists + whitelist_conf['stake_currency'] = base_currency + + ohlcv_history_high_vola = ohlcv_history.copy() + ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090 + + # create candles for medium overall volume with last candle high volume + ohlcv_history_medium_volume = ohlcv_history.copy() + ohlcv_history_medium_volume.loc[ohlcv_history_medium_volume.index == 2, 'volume'] = 5 + + # create candles for high volume with all candles high volume + ohlcv_history_high_volume = ohlcv_history.copy() + ohlcv_history_high_volume.loc[:, 'volume'] = 10 + + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history_medium_volume, + ('XRP/BTC', '1d'): ohlcv_history_high_vola, + ('HOT/BTC', '1d'): ohlcv_history_high_volume, + } + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + if volumefilter_result == 'default_refresh_too_short': + with pytest.raises(OperationalException, + match=r'Refresh period of [0-9]+ seconds is smaller than one timeframe ' + r'of [0-9]+.*\. Please adjust refresh_period to at least [0-9]+ ' + r'and restart the bot\.'): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + return + elif volumefilter_result == 'lookback_days_and_period': + with pytest.raises(OperationalException, + match=r'Ambigous configuration: lookback_days and lookback_period both ' + r'set in pairlist config\..*'): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + elif volumefilter_result == 'lookback_period_negative': + with pytest.raises(OperationalException, + match=r'VolumeFilter requires lookback_period to be >= 0'): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + elif volumefilter_result == 'lookback_exceeds_exchange_request_size': + with pytest.raises(OperationalException, + match=r'VolumeFilter requires lookback_period to not exceed ' + r'exchange max request size \([0-9]+\)'): + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + else: + freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_tickers=tickers, + markets=PropertyMock(return_value=shitcoinmarkets) + ) + + # remove ohlcv when looback_timeframe != 1d + # to enforce fallback to ticker data + if 'lookback_timeframe' in pairlists[0]: + if pairlists[0]['lookback_timeframe'] != '1d': + ohlcv_data = [] + + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), + ) + + freqtrade.pairlists.refresh_pairlist() + whitelist = freqtrade.pairlists.whitelist + + assert isinstance(whitelist, list) + assert whitelist == volumefilter_result + + def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}] del whitelist_conf['stoploss'] @@ -650,6 +778,22 @@ def test_agefilter_min_days_listed_too_small(mocker, default_conf, markets, tick get_patched_freqtradebot(mocker, default_conf) +def test_agefilter_max_days_lower_than_min_days(mocker, default_conf, markets, tickers): + default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, + {'method': 'AgeFilter', 'min_days_listed': 3, + "max_days_listed": 2}] + + mocker.patch.multiple('freqtrade.exchange.Exchange', + markets=PropertyMock(return_value=markets), + exchange_has=MagicMock(return_value=True), + get_tickers=tickers + ) + + with pytest.raises(OperationalException, + match=r'AgeFilter max_days_listed <= min_days_listed not permitted'): + get_patched_freqtradebot(mocker, default_conf) + + def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'AgeFilter', 'min_days_listed': 99999}] @@ -695,6 +839,18 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 +def test_OffsetFilter_error(mocker, whitelist_conf) -> None: + whitelist_conf['pairlists'] = ( + [{"method": "StaticPairList"}, {"method": "OffsetFilter", "offset": -1}] + ) + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + with pytest.raises(OperationalException, + match=r'OffsetFilter requires offset to be >= 0'): + PairListManager(MagicMock, whitelist_conf) + + def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'RangeStabilityFilter', 'lookback_days': 99999}] diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 3d1896a4d..df8679470 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -110,7 +110,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'exchange': 'binance', } - mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) @@ -219,7 +219,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert '-0.41% (-0.06)' == result[0][3] assert '-0.06' == f'{fiat_profit_sum:.2f}' - mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] @@ -429,7 +429,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 @@ -888,7 +888,7 @@ def test_rpcforcebuy(mocker, default_conf, ticker, fee, limit_buy_order_open) -> # Test not buying freqtradebot = get_patched_freqtradebot(mocker, default_conf) - freqtradebot.config['stake_amount'] = 0.0000001 + freqtradebot.config['stake_amount'] = 0 patch_get_signal(freqtradebot, (True, False, '')) rpc = RPC(freqtradebot) pair = 'TKN/BTC' diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 1ab7f178e..afd273998 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -2,6 +2,7 @@ Unit test file for rpc/api_server.py """ +import json from datetime import datetime, timedelta, timezone from pathlib import Path from unittest.mock import ANY, MagicMock, PropertyMock @@ -16,7 +17,7 @@ from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ from freqtrade.enums import RunMode, State -from freqtrade.exceptions import ExchangeError +from freqtrade.exceptions import DependencyException, ExchangeError, OperationalException from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC @@ -48,9 +49,13 @@ def botclient(default_conf, mocker): freqtrade = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(freqtrade) mocker.patch('freqtrade.rpc.api_server.ApiServer.start_api', MagicMock()) - apiserver = ApiServer(rpc, default_conf) - yield freqtrade, TestClient(apiserver.app) - # Cleanup ... ? + try: + apiserver = ApiServer(default_conf) + apiserver.add_rpc_handler(rpc) + yield freqtrade, TestClient(apiserver.app) + # Cleanup ... ? + finally: + ApiServer.shutdown() def client_post(client, url, data={}): @@ -105,6 +110,15 @@ def test_api_ui_fallback(botclient): assert rc.status_code == 200 +def test_api_ui_version(botclient, mocker): + ftbot, client = botclient + + mocker.patch('freqtrade.commands.deploy_commands.read_ui_version', return_value='0.1.2') + rc = client_get(client, "/ui_version") + assert rc.status_code == 200 + assert rc.json()['version'] == '0.1.2' + + def test_api_auth(): with pytest.raises(ValueError): create_token({'identity': {'u': 'Freqtrade'}}, 'secret1234', token_type="NotATokenType") @@ -226,8 +240,13 @@ def test_api__init__(default_conf, mocker): }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) mocker.patch('freqtrade.rpc.api_server.webserver.ApiServer.start_api', MagicMock()) - apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) + apiserver = ApiServer(default_conf) + apiserver.add_rpc_handler(RPC(get_patched_freqtradebot(mocker, default_conf))) assert apiserver._config == default_conf + with pytest.raises(OperationalException, match="RPC Handler already attached."): + apiserver.add_rpc_handler(RPC(get_patched_freqtradebot(mocker, default_conf))) + + ApiServer.shutdown() def test_api_UvicornServer(mocker): @@ -289,15 +308,21 @@ def test_api_run(default_conf, mocker, caplog): }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) - server_mock = MagicMock() + server_inst_mock = MagicMock() + server_inst_mock.run_in_thread = MagicMock() + server_inst_mock.run = MagicMock() + server_mock = MagicMock(return_value=server_inst_mock) mocker.patch('freqtrade.rpc.api_server.webserver.UvicornServer', server_mock) - apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) + apiserver = ApiServer(default_conf) + apiserver.add_rpc_handler(RPC(get_patched_freqtradebot(mocker, default_conf))) assert server_mock.call_count == 1 assert apiserver._config == default_conf apiserver.start_api() assert server_mock.call_count == 2 + assert server_inst_mock.run_in_thread.call_count == 2 + assert server_inst_mock.run.call_count == 0 assert server_mock.call_args_list[0][0][0].host == "127.0.0.1" assert server_mock.call_args_list[0][0][0].port == 8080 assert isinstance(server_mock.call_args_list[0][0][0].app, FastAPI) @@ -316,6 +341,8 @@ def test_api_run(default_conf, mocker, caplog): apiserver.start_api() assert server_mock.call_count == 1 + assert server_inst_mock.run_in_thread.call_count == 1 + assert server_inst_mock.run.call_count == 0 assert server_mock.call_args_list[0][0][0].host == "0.0.0.0" assert server_mock.call_args_list[0][0][0].port == 8089 assert isinstance(server_mock.call_args_list[0][0][0].app, FastAPI) @@ -329,12 +356,24 @@ def test_api_run(default_conf, mocker, caplog): "Please make sure that this is intentional!", caplog) assert log_has_re("SECURITY WARNING - `jwt_secret_key` seems to be default.*", caplog) + server_mock.reset_mock() + apiserver._standalone = True + apiserver.start_api() + assert server_inst_mock.run_in_thread.call_count == 0 + assert server_inst_mock.run.call_count == 1 + + apiserver1 = ApiServer(default_conf) + assert id(apiserver1) == id(apiserver) + + apiserver._standalone = False + # Test crashing API server caplog.clear() mocker.patch('freqtrade.rpc.api_server.webserver.UvicornServer', MagicMock(side_effect=Exception)) apiserver.start_api() assert log_has("Api server failed to start.", caplog) + ApiServer.shutdown() def test_api_cleanup(default_conf, mocker, caplog): @@ -350,11 +389,13 @@ def test_api_cleanup(default_conf, mocker, caplog): server_mock.cleanup = MagicMock() mocker.patch('freqtrade.rpc.api_server.webserver.UvicornServer', server_mock) - apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) + apiserver = ApiServer(default_conf) + apiserver.add_rpc_handler(RPC(get_patched_freqtradebot(mocker, default_conf))) apiserver.cleanup() assert apiserver._server.cleanup.call_count == 1 assert log_has("Stopping API Server", caplog) + ApiServer.shutdown() def test_api_reloadconf(botclient): @@ -668,12 +709,16 @@ def test_api_profit(botclient, mocker, ticker, fee, markets): 'profit_all_ratio_mean': -0.6641100666666667, 'profit_all_percent_sum': -398.47, 'profit_all_ratio_sum': -3.9846604, + 'profit_all_percent': -4.41, + 'profit_all_ratio': -0.044063014216106644, 'profit_closed_coin': 0.00073913, 'profit_closed_fiat': 9.124559849999999, 'profit_closed_ratio_mean': 0.0075, 'profit_closed_percent_mean': 0.75, 'profit_closed_ratio_sum': 0.015, 'profit_closed_percent_sum': 1.5, + 'profit_closed_ratio': 7.391275897987988e-07, + 'profit_closed_percent': 0.0, 'trade_count': 6, 'closed_trade_count': 2, 'winning_trades': 2, @@ -834,7 +879,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'exchange': 'binance', } - mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) rc = client_get(client, f"{BASE_URI}/status") @@ -1208,3 +1253,108 @@ def test_list_available_pairs(botclient): assert rc.json()['length'] == 1 assert rc.json()['pairs'] == ['XRP/ETH'] assert len(rc.json()['pair_interval']) == 1 + + +def test_api_backtesting(botclient, mocker, fee, caplog): + ftbot, client = botclient + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + + # Backtesting not started yet + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) + + result = rc.json() + assert result['status'] == 'not_started' + assert not result['running'] + assert result['status_msg'] == 'Backtest not yet executed' + assert result['progress'] == 0 + + # Reset backtesting + rc = client_delete(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'reset' + assert not result['running'] + assert result['status_msg'] == 'Backtest reset' + + # start backtesting + data = { + "strategy": "DefaultStrategy", + "timeframe": "5m", + "timerange": "20180110-20180111", + "max_open_trades": 3, + "stake_amount": 100, + "dry_run_wallet": 1000, + "enable_protections": False + } + rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) + assert_response(rc) + result = rc.json() + + assert result['status'] == 'running' + assert result['progress'] == 0 + assert result['running'] + assert result['status_msg'] == 'Backtest started' + + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) + + result = rc.json() + assert result['status'] == 'ended' + assert not result['running'] + assert result['status_msg'] == 'Backtest ended' + assert result['progress'] == 1 + assert result['backtest_result'] + + rc = client_get(client, f"{BASE_URI}/backtest/abort") + assert_response(rc) + result = rc.json() + assert result['status'] == 'not_running' + assert not result['running'] + assert result['status_msg'] == 'Backtest ended' + + # Simulate running backtest + ApiServer._bgtask_running = True + rc = client_get(client, f"{BASE_URI}/backtest/abort") + assert_response(rc) + result = rc.json() + assert result['status'] == 'stopping' + assert not result['running'] + assert result['status_msg'] == 'Backtest ended' + + # Get running backtest... + rc = client_get(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'running' + assert result['running'] + assert result['step'] == "backtest" + assert result['status_msg'] == "Backtest running" + + # Try delete with task still running + rc = client_delete(client, f"{BASE_URI}/backtest") + assert_response(rc) + result = rc.json() + assert result['status'] == 'running' + + # Post to backtest that's still running + rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) + assert_response(rc, 502) + result = rc.json() + assert 'Bot Background task already running' in result['error'] + + ApiServer._bgtask_running = False + + mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest_one_strategy', + side_effect=DependencyException()) + rc = client_post(client, f"{BASE_URI}/backtest", data=json.dumps(data)) + assert log_has("Backtesting caused an error: ", caplog) + + # Delete backtesting to avoid leakage since the backtest-object may stick around. + rc = client_delete(client, f"{BASE_URI}/backtest") + assert_response(rc) + + result = rc.json() + assert result['status'] == 'reset' + assert not result['running'] + assert result['status_msg'] == 'Backtest reset' diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index 918022386..596b5ae20 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -5,6 +5,7 @@ from unittest.mock import MagicMock from freqtrade.enums import RPCMessageType from freqtrade.rpc import RPCManager +from freqtrade.rpc.api_server.webserver import ApiServer from tests.conftest import get_patched_freqtradebot, log_has @@ -190,3 +191,4 @@ def test_init_apiserver_enabled(mocker, default_conf, caplog) -> None: assert len(rpc_manager.registered_modules) == 1 assert 'apiserver' in [mod.name for mod in rpc_manager.registered_modules] assert run_mock.call_count == 1 + ApiServer.shutdown() diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 642b55975..df624e136 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -452,7 +452,8 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, assert msg_mock.call_count == 1 assert 'No closed trade' in msg_mock.call_args_list[-1][0][0] assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] - assert ('∙ `-0.00000500 BTC (-0.50%) (-0.5 \N{GREEK CAPITAL LETTER SIGMA}%)`' + mocker.patch('freqtrade.wallets.Wallets.get_starting_balance', return_value=0.01) + assert ('∙ `-0.00000500 BTC (-0.50%) (-0.0 \N{GREEK CAPITAL LETTER SIGMA}%)`' in msg_mock.call_args_list[-1][0][0]) msg_mock.reset_mock() @@ -466,11 +467,11 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, telegram._profit(update=update, context=MagicMock()) assert msg_mock.call_count == 1 assert '*ROI:* Closed trades' in msg_mock.call_args_list[-1][0][0] - assert ('∙ `0.00006217 BTC (6.20%) (6.2 \N{GREEK CAPITAL LETTER SIGMA}%)`' + assert ('∙ `0.00006217 BTC (6.20%) (0.62 \N{GREEK CAPITAL LETTER SIGMA}%)`' in msg_mock.call_args_list[-1][0][0]) assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] - assert ('∙ `0.00006217 BTC (6.20%) (6.2 \N{GREEK CAPITAL LETTER SIGMA}%)`' + assert ('∙ `0.00006217 BTC (6.20%) (0.62 \N{GREEK CAPITAL LETTER SIGMA}%)`' in msg_mock.call_args_list[-1][0][0]) assert '∙ `0.933 USD`' in msg_mock.call_args_list[-1][0][0] @@ -519,12 +520,15 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick assert msg_mock.call_count == 1 assert '*BTC:*' in result assert '*ETH:*' not in result - assert '*USDT:*' in result - assert '*EUR:*' in result + assert '*USDT:*' not in result + assert '*EUR:*' not in result + assert '*LTC:*' in result + assert '*XRP:*' not in result assert 'Balance:' in result assert 'Est. BTC:' in result assert 'BTC: 12.00000000' in result - assert '*XRP:* not showing <0.0001 BTC amount' in result + assert "*3 Other Currencies (< 0.0001 BTC):*" in result + assert 'BTC: 0.00000309' in result def test_balance_handle_empty_response(default_conf, update, mocker) -> None: diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index c02a43e72..aec07266d 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, C0103 import logging from datetime import datetime, timedelta, timezone +from pathlib import Path from unittest.mock import MagicMock import arrow @@ -712,3 +713,50 @@ def test_auto_hyperopt_interface(default_conf): with pytest.raises(OperationalException, match=r"Inconclusive parameter.*"): [x for x in strategy.detect_parameters('sell')] + + +def test_auto_hyperopt_interface_loadparams(default_conf, mocker, caplog): + default_conf.update({'strategy': 'HyperoptableStrategy'}) + del default_conf['stoploss'] + del default_conf['minimal_roi'] + mocker.patch.object(Path, 'is_file', MagicMock(return_value=True)) + mocker.patch.object(Path, 'open') + expected_result = { + "strategy_name": "HyperoptableStrategy", + "params": { + "stoploss": { + "stoploss": -0.05, + }, + "roi": { + "0": 0.2, + "1200": 0.01 + } + } + } + mocker.patch('freqtrade.strategy.hyper.json_load', return_value=expected_result) + PairLocks.timeframe = default_conf['timeframe'] + strategy = StrategyResolver.load_strategy(default_conf) + assert strategy.stoploss == -0.05 + assert strategy.minimal_roi == {0: 0.2, 1200: 0.01} + + expected_result = { + "strategy_name": "HyperoptableStrategy_No", + "params": { + "stoploss": { + "stoploss": -0.05, + }, + "roi": { + "0": 0.2, + "1200": 0.01 + } + } + } + + mocker.patch('freqtrade.strategy.hyper.json_load', return_value=expected_result) + with pytest.raises(OperationalException, match="Invalid parameter file provided."): + StrategyResolver.load_strategy(default_conf) + + mocker.patch('freqtrade.strategy.hyper.json_load', MagicMock(side_effect=ValueError())) + + StrategyResolver.load_strategy(default_conf) + assert log_has("Invalid parameter file format.", caplog) diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 0d81dea28..fd6f162fd 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -172,7 +172,7 @@ def test_download_data_options() -> None: def test_plot_dataframe_options() -> None: args = [ 'plot-dataframe', - '-c', 'config_bittrex.json.example', + '-c', 'config_examples/config_bittrex.example.json', '--indicators1', 'sma10', 'sma100', '--indicators2', 'macd', 'fastd', 'fastk', '--plot-limit', '30', diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 8edd09c5a..34db892b2 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -28,7 +28,7 @@ from tests.conftest import log_has, log_has_re, patched_configuration_load_confi @pytest.fixture(scope="function") def all_conf(): - config_file = Path(__file__).parents[1] / "config_full.json.example" + config_file = Path(__file__).parents[1] / "config_examples/config_full.example.json" conf = load_config_file(str(config_file)) return conf diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 583ec7ddd..48a0f06e8 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -161,7 +161,7 @@ def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: (True, 0.0022, 3, 0.5, [0.001, 0.001, 0.0]), (True, 0.0027, 3, 0.5, [0.001, 0.001, 0.000673]), (True, 0.0022, 3, 1, [0.001, 0.001, 0.0]), - ]) +]) def test_check_available_stake_amount(default_conf, ticker, mocker, fee, limit_buy_order_open, amend_last, wallet, max_open, lsamr, expected) -> None: patch_RPCManager(mocker) @@ -397,7 +397,7 @@ def test_create_trade_minimal_amount(default_conf, ticker, limit_buy_order_open, def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_order_open, - fee, mocker) -> None: + fee, mocker, caplog) -> None: patch_RPCManager(mocker) patch_exchange(mocker) buy_mock = MagicMock(return_value=limit_buy_order_open) @@ -413,6 +413,27 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord patch_get_signal(freqtrade) + assert freqtrade.create_trade('ETH/BTC') + assert log_has_re(r"Stake amount for pair .* is too small.*", caplog) + + +def test_create_trade_zero_stake_amount(default_conf, ticker, limit_buy_order_open, + fee, mocker) -> None: + patch_RPCManager(mocker) + patch_exchange(mocker) + buy_mock = MagicMock(return_value=limit_buy_order_open) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + buy=buy_mock, + get_fee=fee, + ) + + freqtrade = FreqtradeBot(default_conf) + freqtrade.config['stake_amount'] = 0 + + patch_get_signal(freqtrade) + assert not freqtrade.create_trade('ETH/BTC') @@ -763,7 +784,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order buy_mm = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_buy_rate=buy_rate_mock, + get_rate=buy_rate_mock, fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, @@ -803,7 +824,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order limit_buy_order_open['id'] = '33' fix_price = 0.06 assert freqtrade.execute_buy(pair, stake_amount, fix_price) - # Make sure get_buy_rate wasn't called again + # Make sure get_rate wasn't called again assert buy_rate_mock.call_count == 0 assert buy_mm.call_count == 2 @@ -842,6 +863,24 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order assert trade.open_rate == 0.5 assert trade.stake_amount == 40.495905365 + # Test with custom stake + limit_buy_order['status'] = 'open' + limit_buy_order['id'] = '556' + + freqtrade.strategy.custom_stake_amount = lambda **kwargs: 150.0 + assert freqtrade.execute_buy(pair, stake_amount) + trade = Trade.query.all()[4] + assert trade + assert trade.stake_amount == 150 + + # Exception case + limit_buy_order['id'] = '557' + freqtrade.strategy.custom_stake_amount = lambda **kwargs: 20 / 0 + assert freqtrade.execute_buy(pair, stake_amount) + trade = Trade.query.all()[5] + assert trade + assert trade.stake_amount == 2.0 + # In case of the order is rejected and not filled at all limit_buy_order['status'] = 'rejected' limit_buy_order['amount'] = 90.99181073 @@ -854,7 +893,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order assert not freqtrade.execute_buy(pair, stake_amount) # Fail to get price... - mocker.patch('freqtrade.exchange.Exchange.get_buy_rate', MagicMock(return_value=0.0)) + mocker.patch('freqtrade.exchange.Exchange.get_rate', MagicMock(return_value=0.0)) with pytest.raises(PricingError, match="Could not determine buy price."): freqtrade.execute_buy(pair, stake_amount) @@ -870,7 +909,7 @@ def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) - 'last': 0.00001172 }), buy=MagicMock(return_value=limit_buy_order), - get_buy_rate=MagicMock(return_value=0.11), + get_rate=MagicMock(return_value=0.11), get_min_pair_stake_amount=MagicMock(return_value=1), get_fee=fee, ) @@ -2474,7 +2513,7 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: 'freqtrade.exchange.Exchange', cancel_order=cancel_order_mock, ) - mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', return_value=0.245441) + mocker.patch('freqtrade.exchange.Exchange.get_rate', return_value=0.245441) freqtrade = FreqtradeBot(default_conf) @@ -3917,7 +3956,7 @@ def test_order_book_depth_of_market_high_delta(default_conf, ticker, limit_buy_o def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: """ - test if function get_buy_rate will return the order book price + test if function get_rate will return the order book price instead of the ask rate """ patch_exchange(mocker) @@ -3935,7 +3974,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.exchange.get_buy_rate('ETH/BTC', True) == 0.043935 + assert freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") == 0.043935 assert ticker_mock.call_count == 0 @@ -3957,8 +3996,8 @@ def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None freqtrade = FreqtradeBot(default_conf) # orderbook shall be used even if tickers would be lower. with pytest.raises(PricingError): - freqtrade.exchange.get_buy_rate('ETH/BTC', refresh=True) - assert log_has_re(r'Buy Price from orderbook could not be determined.', caplog) + freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") + assert log_has_re(r'Buy Price at location 1 from orderbook could not be determined.', caplog) def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: diff --git a/tests/test_main.py b/tests/test_main.py index 3546a3bab..59a5bb0f7 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -67,12 +67,12 @@ def test_main_fatal_exception(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = ['trade', '-c', 'config_bittrex.json.example'] + args = ['trade', '-c', 'config_examples/config_bittrex.example.json'] # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit): main(args) - assert log_has('Using config: config_bittrex.json.example ...', caplog) + assert log_has('Using config: config_examples/config_bittrex.example.json ...', caplog) assert log_has('Fatal exception!', caplog) @@ -85,12 +85,12 @@ def test_main_keyboard_interrupt(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.wallets.Wallets.update', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = ['trade', '-c', 'config_bittrex.json.example'] + args = ['trade', '-c', 'config_examples/config_bittrex.example.json'] # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit): main(args) - assert log_has('Using config: config_bittrex.json.example ...', caplog) + assert log_has('Using config: config_examples/config_bittrex.example.json ...', caplog) assert log_has('SIGINT received, aborting ...', caplog) @@ -106,12 +106,12 @@ def test_main_operational_exception(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = ['trade', '-c', 'config_bittrex.json.example'] + args = ['trade', '-c', 'config_examples/config_bittrex.example.json'] # Test Main + the KeyboardInterrupt exception with pytest.raises(SystemExit): main(args) - assert log_has('Using config: config_bittrex.json.example ...', caplog) + assert log_has('Using config: config_examples/config_bittrex.example.json ...', caplog) assert log_has('Oh snap!', caplog) @@ -157,12 +157,16 @@ def test_main_reload_config(mocker, default_conf, caplog) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = Arguments(['trade', '-c', 'config_bittrex.json.example']).get_parsed_arg() + args = Arguments([ + 'trade', + '-c', + 'config_examples/config_bittrex.example.json' + ]).get_parsed_arg() worker = Worker(args=args, config=default_conf) with pytest.raises(SystemExit): - main(['trade', '-c', 'config_bittrex.json.example']) + main(['trade', '-c', 'config_examples/config_bittrex.example.json']) - assert log_has('Using config: config_bittrex.json.example ...', caplog) + assert log_has('Using config: config_examples/config_bittrex.example.json ...', caplog) assert worker_mock.call_count == 4 assert reconfigure_mock.call_count == 1 assert isinstance(worker.freqtrade, FreqtradeBot) @@ -180,7 +184,11 @@ def test_reconfigure(mocker, default_conf) -> None: mocker.patch('freqtrade.freqtradebot.RPCManager', MagicMock()) mocker.patch('freqtrade.freqtradebot.init_db', MagicMock()) - args = Arguments(['trade', '-c', 'config_bittrex.json.example']).get_parsed_arg() + args = Arguments([ + 'trade', + '-c', + 'config_examples/config_bittrex.example.json' + ]).get_parsed_arg() worker = Worker(args=args, config=default_conf) freqtrade = worker.freqtrade diff --git a/tests/test_misc.py b/tests/test_misc.py index e6ba70aee..221c7b712 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock import pytest from freqtrade.misc import (decimals_per_coin, file_dump_json, file_load_json, format_ms_time, - pair_to_filename, plural, render_template, + pair_to_filename, parse_db_uri_for_logging, plural, render_template, render_template_with_fallback, round_coin_value, safe_value_fallback, safe_value_fallback2, shorten_date) @@ -179,3 +179,18 @@ def test_render_template_fallback(mocker): ) assert isinstance(val, str) assert 'if self.dp' in val + + +def test_parse_db_uri_for_logging() -> None: + postgresql_conn_uri = "postgresql+psycopg2://scott123:scott123@host/dbname" + mariadb_conn_uri = "mariadb+mariadbconnector://app_user:Password123!@127.0.0.1:3306/company" + mysql_conn_uri = "mysql+pymysql://user:pass@some_mariadb/dbname?charset=utf8mb4" + sqlite_conn_uri = "sqlite:////freqtrade/user_data/tradesv3.sqlite" + censored_pwd = "*****" + + def get_pwd(x): return x.split(':')[2].split('@')[0] + + assert get_pwd(parse_db_uri_for_logging(postgresql_conn_uri)) == censored_pwd + assert get_pwd(parse_db_uri_for_logging(mariadb_conn_uri)) == censored_pwd + assert get_pwd(parse_db_uri_for_logging(mysql_conn_uri)) == censored_pwd + assert sqlite_conn_uri == parse_db_uri_for_logging(sqlite_conn_uri) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 1576aaa5a..89d07ca74 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1124,6 +1124,21 @@ def test_total_open_trades_stakes(fee, use_db): Trade.use_db = True +@pytest.mark.usefixtures("init_persistence") +@pytest.mark.parametrize('use_db', [True, False]) +def test_get_total_closed_profit(fee, use_db): + + Trade.use_db = use_db + Trade.reset_trades() + res = Trade.get_total_closed_profit() + assert res == 0 + create_mock_trades(fee, use_db) + res = Trade.get_total_closed_profit() + assert res == 0.000739127 + + Trade.use_db = True + + @pytest.mark.usefixtures("init_persistence") @pytest.mark.parametrize('use_db', [True, False]) def test_get_trades_proxy(fee, use_db): @@ -1298,6 +1313,7 @@ def test_Trade_object_idem(): 'open_date', 'get_best_pair', 'get_overall_performance', + 'get_total_closed_profit', 'total_open_trades_stakes', 'get_sold_trades_without_assigned_fees', 'get_open_trades_without_assigned_fees', diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 20f159e3a..ecadc3f8b 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -364,7 +364,7 @@ def test_start_plot_dataframe(mocker): aup = mocker.patch("freqtrade.plot.plotting.load_and_plot_trades", MagicMock()) args = [ "plot-dataframe", - "--config", "config_bittrex.json.example", + "--config", "config_examples/config_bittrex.example.json", "--pairs", "ETH/BTC" ] start_plot_dataframe(get_args(args)) @@ -408,7 +408,7 @@ def test_start_plot_profit(mocker): aup = mocker.patch("freqtrade.plot.plotting.plot_profit", MagicMock()) args = [ "plot-profit", - "--config", "config_bittrex.json.example", + "--config", "config_examples/config_bittrex.example.json", "--pairs", "ETH/BTC" ] start_plot_profit(get_args(args)) diff --git a/tests/test_wallets.py b/tests/test_wallets.py index ff303e2ec..64db3b9cd 100644 --- a/tests/test_wallets.py +++ b/tests/test_wallets.py @@ -121,13 +121,19 @@ def test_get_trade_stake_amount_no_stake_amount(default_conf, mocker) -> None: freqtrade.wallets.get_trade_stake_amount('ETH/BTC') -@pytest.mark.parametrize("balance_ratio,result1,result2", [ - (1, 50, 66.66666), - (0.99, 49.5, 66.0), - (0.50, 25, 33.3333), +@pytest.mark.parametrize("balance_ratio,capital,result1,result2", [ + (1, None, 50, 66.66666), + (0.99, None, 49.5, 66.0), + (0.50, None, 25, 33.3333), + # Tests with capital ignore balance_ratio + (1, 100, 50, 0.0), + (0.99, 200, 50, 66.66666), + (0.99, 150, 50, 50), + (0.50, 50, 25, 0.0), + (0.50, 10, 5, 0.0), ]) -def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_ratio, result1, - result2, limit_buy_order_open, +def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_ratio, capital, + result1, result2, limit_buy_order_open, fee, mocker) -> None: mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -141,6 +147,8 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_r conf['dry_run_wallet'] = 100 conf['max_open_trades'] = 2 conf['tradable_balance_ratio'] = balance_ratio + if capital is not None: + conf['available_capital'] = capital freqtrade = get_patched_freqtradebot(mocker, conf) @@ -170,3 +178,49 @@ def test_get_trade_stake_amount_unlimited_amount(default_conf, ticker, balance_r freqtrade.config['max_open_trades'] = 0 result = freqtrade.wallets.get_trade_stake_amount('NEO/USDT') assert result == 0 + + +@pytest.mark.parametrize('stake_amount,min_stake_amount,max_stake_amount,expected', [ + (22, 11, 50, 22), + (100, 11, 500, 100), + (1000, 11, 500, 500), # Above max-stake + (20, 15, 10, 0), # Minimum stake > max-stake + (1, 11, 100, 11), # Below min stake + (1, 15, 10, 0), # Below min stake and min_stake > max_stake + +]) +def test__validate_stake_amount(mocker, default_conf, + stake_amount, min_stake_amount, max_stake_amount, expected): + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + mocker.patch("freqtrade.wallets.Wallets.get_available_stake_amount", + return_value=max_stake_amount) + res = freqtrade.wallets._validate_stake_amount('XRP/USDT', stake_amount, min_stake_amount) + assert res == expected + + +@pytest.mark.parametrize('available_capital,closed_profit,open_stakes,free,expected', [ + (None, 10, 100, 910, 1000), + (None, 0, 0, 2500, 2500), + (None, 500, 0, 2500, 2000), + (None, 500, 0, 2500, 2000), + (None, -70, 0, 1930, 2000), + # Only available balance matters when it's set. + (100, 0, 0, 0, 100), + (1000, 0, 2, 5, 1000), + (1235, 2250, 2, 5, 1235), + (1235, -2250, 2, 5, 1235), +]) +def test_get_starting_balance(mocker, default_conf, available_capital, closed_profit, + open_stakes, free, expected): + if available_capital: + default_conf['available_capital'] = available_capital + mocker.patch("freqtrade.persistence.models.Trade.get_total_closed_profit", + return_value=closed_profit) + mocker.patch("freqtrade.persistence.models.Trade.total_open_trades_stakes", + return_value=open_stakes) + mocker.patch("freqtrade.wallets.Wallets.get_free", return_value=free) + + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + assert freqtrade.wallets.get_starting_balance() == expected