diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f259129d4..daa10fea7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,12 +14,109 @@ on: - cron: '0 5 * * 4' jobs: - build: + build_linux: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-18.04, ubuntu-20.04, macos-latest ] + os: [ ubuntu-18.04, ubuntu-20.04 ] + python-version: [3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache_dependencies + uses: actions/cache@v2 + id: cache + with: + path: ~/dependencies/ + key: ${{ runner.os }}-dependencies + + - name: pip cache (linux) + uses: actions/cache@v2 + if: startsWith(matrix.os, 'ubuntu') + with: + path: ~/.cache/pip + key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip + + - name: TA binary *nix + if: steps.cache.outputs.cache-hit != 'true' + run: | + cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. + + - name: Installation - *nix + run: | + python -m pip install --upgrade pip + export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH + export TA_LIBRARY_PATH=${HOME}/dependencies/lib + export TA_INCLUDE_PATH=${HOME}/dependencies/include + pip install -r requirements-dev.txt + pip install -e . + + - name: Tests + run: | + pytest --random-order --cov=freqtrade --cov-config=.coveragerc + if: matrix.python-version != '3.9' + + - name: Tests incl. ccxt compatibility tests + run: | + pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun + if: matrix.python-version == '3.9' + + - name: Coveralls + if: (startsWith(matrix.os, 'ubuntu-20') && matrix.python-version == '3.8') + env: + # Coveralls token. Not used as secret due to github not providing secrets to forked repositories + COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu + run: | + # Allow failure for coveralls + coveralls -v || true + + - name: Backtesting + run: | + cp config.json.example config.json + freqtrade create-userdir --userdir user_data + freqtrade backtesting --datadir tests/testdata --strategy SampleStrategy + + - name: Hyperopt + run: | + cp config.json.example config.json + freqtrade create-userdir --userdir user_data + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + + - name: Flake8 + run: | + flake8 + + - name: Sort imports (isort) + run: | + isort --check . + + - name: Mypy + run: | + mypy freqtrade scripts + + - name: Slack Notification + uses: homoluctus/slatify@v1.8.0 + if: failure() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) + with: + type: ${{ job.status }} + job_name: '*Freqtrade CI ${{ matrix.os }}*' + mention: 'here' + mention_if: 'failure' + channel: '#notifications' + url: ${{ secrets.SLACK_WEBHOOK }} + + build_macos: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ macos-latest ] python-version: [3.7, 3.8] steps: @@ -31,21 +128,14 @@ jobs: python-version: ${{ matrix.python-version }} - name: Cache_dependencies - uses: actions/cache@v1 + uses: actions/cache@v2 id: cache with: path: ~/dependencies/ key: ${{ runner.os }}-dependencies - - name: pip cache (linux) - uses: actions/cache@preview - if: startsWith(matrix.os, 'ubuntu') - with: - path: ~/.cache/pip - key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip - - name: pip cache (macOS) - uses: actions/cache@preview + uses: actions/cache@v2 if: startsWith(matrix.os, 'macOS') with: path: ~/Library/Caches/pip @@ -113,6 +203,7 @@ jobs: channel: '#notifications' url: ${{ secrets.SLACK_WEBHOOK }} + build_windows: runs-on: ${{ matrix.os }} @@ -215,7 +306,7 @@ jobs: # Notify on slack only once - when CI completes (and after deploy) in case it's successfull notify-complete: - needs: [ build, build_windows, docs_check ] + needs: [ build_linux, build_macos, build_windows, docs_check ] runs-on: ubuntu-20.04 steps: - name: Slack Notification @@ -228,8 +319,9 @@ jobs: url: ${{ secrets.SLACK_WEBHOOK }} deploy: - needs: [ build, build_windows, docs_check ] + needs: [ build_linux, build_macos, build_windows, docs_check ] runs-on: ubuntu-20.04 + if: (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'release') && github.repository == 'freqtrade/freqtrade' steps: - uses: actions/checkout@v2 diff --git a/.readthedocs.yml b/.readthedocs.yml index dec7b44d7..446181452 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -4,5 +4,5 @@ build: image: latest python: - version: 3.6 - setup_py_install: false \ No newline at end of file + version: 3.8 + setup_py_install: false diff --git a/.travis.yml b/.travis.yml index 9b8448db5..94239e33f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ os: - linux -dist: xenial +dist: bionic language: python python: -- 3.6 +- 3.8 services: - docker env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b4e8adaf..5c52a8e93 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Few pointers for contributions: - New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR. - PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished). -If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. +If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. ## Getting started diff --git a/Dockerfile b/Dockerfile index 2be65274e..602e6a28c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,41 @@ -FROM python:3.8.6-slim-buster +FROM python:3.8.6-slim-buster as base -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev sqlite3 \ - && apt-get clean \ - && pip install --upgrade pip +# Setup env +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONFAULTHANDLER 1 +ENV PATH=/root/.local/bin:$PATH # Prepare environment RUN mkdir /freqtrade WORKDIR /freqtrade +# Install dependencies +FROM base as python-deps +RUN apt-get update \ + && apt-get -y install curl build-essential libssl-dev git \ + && apt-get clean \ + && pip install --upgrade pip + # Install TA-lib COPY build_helpers/* /tmp/ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* - ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies COPY requirements.txt requirements-hyperopt.txt /freqtrade/ -RUN pip install numpy --no-cache-dir \ - && pip install -r requirements-hyperopt.txt --no-cache-dir +RUN pip install --user --no-cache-dir numpy \ + && pip install --user --no-cache-dir -r requirements-hyperopt.txt + +# Copy dependencies to runtime-image +FROM base as runtime-image +COPY --from=python-deps /usr/local/lib /usr/local/lib +ENV LD_LIBRARY_PATH /usr/local/lib + +COPY --from=python-deps /root/.local /root/.local + + # Install and execute COPY . /freqtrade/ diff --git a/Dockerfile.armhf b/Dockerfile.armhf index 0633008ea..b6f2e44e6 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -1,25 +1,43 @@ -FROM --platform=linux/arm/v7 python:3.7.7-slim-buster +FROM --platform=linux/arm/v7 python:3.7.9-slim-buster as base -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev libffi-dev libatlas3-base libgfortran5 sqlite3 \ - && apt-get clean \ - && pip install --upgrade pip \ - && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf +# Setup env +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONFAULTHANDLER 1 +ENV PATH=/root/.local/bin:$PATH # Prepare environment RUN mkdir /freqtrade WORKDIR /freqtrade +RUN apt-get update \ + && apt-get -y install libatlas3-base curl sqlite3 \ + && apt-get clean + +# Install dependencies +FROM base as python-deps +RUN apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \ + && apt-get clean \ + && pip install --upgrade pip \ + && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf + # Install TA-lib COPY build_helpers/* /tmp/ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* - ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies COPY requirements.txt /freqtrade/ -RUN pip install numpy --no-cache-dir \ - && pip install -r requirements.txt --no-cache-dir +RUN pip install --user --no-cache-dir numpy \ + && pip install --user --no-cache-dir -r requirements.txt + +# Copy dependencies to runtime-image +FROM base as runtime-image +COPY --from=python-deps /usr/local/lib /usr/local/lib +ENV LD_LIBRARY_PATH /usr/local/lib + +COPY --from=python-deps /root/.local /root/.local # Install and execute COPY . /freqtrade/ diff --git a/README.md b/README.md index 8526b5c91..1031e4d67 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Please find the complete documentation on our [website](https://www.freqtrade.io ## Features -- [x] **Based on Python 3.6+**: For botting on any operating system - Windows, macOS and Linux. +- [x] **Based on Python 3.7+**: For botting on any operating system - Windows, macOS and Linux. - [x] **Persistence**: Persistence is achieved through sqlite. - [x] **Dry-run**: Run the bot without playing money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. @@ -138,7 +138,7 @@ For any questions not covered by the documentation or for further information ab Please check out our [discord server](https://discord.gg/MA9v74M). -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg). +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA). ### [Bugs / Issues](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue) @@ -169,7 +169,7 @@ to understand the requirements before sending your pull-requests. Coding is not a necessity to contribute - maybe start with improving our documentation? Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. -**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/MA9v74M) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/MA9v74M) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. **Important:** Always create your PR against the `develop` branch, not `stable`. @@ -187,7 +187,7 @@ To run this bot we recommend you a cloud instance with a minimum of: ### Software requirements -- [Python 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/) +- [Python 3.7.x](http://docs.python-guide.org/en/latest/starting/installation/) - [pip](https://pip.pypa.io/en/stable/installing/) - [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) - [TA-Lib](https://mrjbq7.github.io/ta-lib/install.html) diff --git a/config_full.json.example b/config_full.json.example index 5ee2a1faf..e69e52469 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -75,6 +75,33 @@ "refresh_period": 1440 } ], + "protections": [ + { + "method": "StoplossGuard", + "lookback_period_candles": 60, + "trade_limit": 4, + "stop_duration_candles": 60, + "only_per_pair": false + }, + { + "method": "CooldownPeriod", + "stop_duration_candles": 20 + }, + { + "method": "MaxDrawdown", + "lookback_period_candles": 200, + "trade_limit": 20, + "stop_duration_candles": 10, + "max_allowed_drawdown": 0.2 + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 360, + "trade_limit": 1, + "stop_duration_candles": 2, + "required_profit": 0.02 + } + ], "exchange": { "name": "bittrex", "sandbox": false, diff --git a/docker-compose.yml b/docker-compose.yml index a99aac3c7..7094500b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: # Build step - only needed when additional dependencies are needed # build: # context: . - # dockerfile: "./Dockerfile.technical" + # dockerfile: "./docker/Dockerfile.technical" restart: unless-stopped container_name: freqtrade volumes: diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 59ebc16b5..1ace61769 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -77,7 +77,7 @@ Currently, the arguments are: * `results`: DataFrame containing the result The following columns are available in results (corresponds to the output-file of backtesting when used with `--export trades`): - `pair, profit_percent, profit_abs, open_time, close_time, open_index, close_index, trade_duration, open_at_end, open_rate, close_rate, sell_reason` + `pair, profit_percent, profit_abs, open_date, open_rate, open_fee, close_date, close_rate, close_fee, amount, trade_duration, open_at_end, sell_reason` * `trade_count`: Amount of trades (identical to `len(results)`) * `min_date`: Start date of the hyperopting TimeFrame * `min_date`: End date of the hyperopting TimeFrame diff --git a/docs/backtesting.md b/docs/backtesting.md index 277b11083..27bfebe37 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -165,10 +165,13 @@ A backtesting result will look like that: | Max open trades | 3 | | | | | Total trades | 429 | -| First trade | 2019-01-01 18:30:00 | -| First trade Pair | EOS/USDT | | Total Profit % | 152.41% | | Trades per day | 3.575 | +| | | +| Best Pair | LSK/BTC 26.26% | +| Worst Pair | ZEC/BTC -10.18% | +| Best Trade | LSK/BTC 4.25% | +| Worst Trade | ZEC/BTC -10.25% | | Best day | 25.27% | | Worst day | -30.67% | | Avg. Duration Winners | 4:23:00 | @@ -238,10 +241,13 @@ It contains some useful key metrics about performance of your strategy on backte | Max open trades | 3 | | | | | Total trades | 429 | -| First trade | 2019-01-01 18:30:00 | -| First trade Pair | EOS/USDT | | Total Profit % | 152.41% | | Trades per day | 3.575 | +| | | +| Best Pair | LSK/BTC 26.26% | +| Worst Pair | ZEC/BTC -10.18% | +| Best Trade | LSK/BTC 4.25% | +| Worst Trade | ZEC/BTC -10.25% | | Best day | 25.27% | | Worst day | -30.67% | | Avg. Duration Winners | 4:23:00 | @@ -258,10 +264,10 @@ It contains some useful key metrics about performance of your strategy on backte - `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option). - `Max open trades`: Setting of `max_open_trades` (or `--max-open-trades`) - to clearly see settings for this. - `Total trades`: Identical to the total trades of the backtest output table. -- `First trade`: First trade entered. -- `First trade pair`: Which pair was part of the first trade. - `Total Profit %`: Total profit per stake amount. Aligned to the TOTAL column of the first table. - `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). +- `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. +- `Best Trade` / `Worst Trade`: Biggest winning trade and biggest losing trade - `Best day` / `Worst day`: Best and worst day based on daily profit. - `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades. - `Max Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced). @@ -273,18 +279,24 @@ It contains some useful key metrics about performance of your strategy on backte Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions: - Buys happen at open-price -- Sell signal sells happen at open-price of the following candle -- Low happens before high for stoploss, protecting capital first +- Sell-signal sells happen at open-price of the consecutive candle +- Sell-signal is favored over Stoploss, because sell-signals are assumed to trigger on candle's open - ROI - sells are compared to high - but the ROI value is used (e.g. ROI = 2%, high=5% - so the sell will be at 2%) - sells are never "below the candle", so a ROI of 2% may result in a sell at 2.4% if low was at 2.4% profit - Forcesells caused by `=-1` ROI entries use low as sell value, unless N falls on the candle open (e.g. `120: -1` for 1h candles) -- Stoploss sells happen exactly at stoploss price, even if low was lower +- Stoploss sells happen exactly at stoploss price, even if low was lower, but the loss will be `2 * fees` higher than the stoploss price +- Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes +- Low happens before high for stoploss, protecting capital first - Trailing stoploss - High happens first - adjusting stoploss - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly) + - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies - Sell-reason does not explain if a trade was positive or negative, just what triggered the sell (this can look odd if negative ROI values are used) -- Stoploss (and trailing stoploss) is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` and/or `trailing_stop` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes. +- Evaluation sequence (if multiple signals happen on the same candle) + - ROI (if not stoploss) + - Sell-signal + - Stoploss Taking these assumptions, backtesting tries to mirror real trading as closely as possible. However, backtesting will **never** replace running a strategy in dry-run mode. Also, keep in mind that past results don't guarantee future success. diff --git a/docs/bot-usage.md b/docs/bot-usage.md index 4d07435c7..5820b3cc7 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -213,9 +213,11 @@ Backtesting also uses the config specified via `-c/--config`. usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TIMEFRAME] - [--timerange TIMERANGE] [--max-open-trades INT] + [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] + [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] - [--eps] [--dmmp] + [--eps] [--dmmp] [--enable-protections] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] [--export EXPORT] [--export-filename PATH] @@ -226,6 +228,9 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `None`). --max-open-trades INT Override the value of the `max_open_trades` configuration setting. @@ -241,6 +246,10 @@ optional arguments: Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number). + --enable-protections, --enableprotections + Enable protections for backtesting.Will slow + backtesting down by a considerable amount, but will + include configured protections --strategy-list STRATEGY_LIST [STRATEGY_LIST ...] Provide a space-separated list of strategies to backtest. Please note that ticker-interval needs to be @@ -296,13 +305,14 @@ to find optimal parameter values for your strategy. usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [-s NAME] [--strategy-path PATH] [-i TIMEFRAME] [--timerange TIMERANGE] + [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] [--hyperopt NAME] [--hyperopt-path PATH] [--eps] - [-e INT] + [--dmmp] [--enable-protections] [-e INT] [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]] - [--dmmp] [--print-all] [--no-color] [--print-json] - [-j JOBS] [--random-state INT] [--min-trades INT] + [--print-all] [--no-color] [--print-json] [-j JOBS] + [--random-state INT] [--min-trades INT] [--hyperopt-loss NAME] optional arguments: @@ -312,6 +322,9 @@ optional arguments: `1d`). --timerange TIMERANGE Specify what timerange of data to use. + --data-format-ohlcv {json,jsongz,hdf5} + Storage format for downloaded candle (OHLCV) data. + (default: `None`). --max-open-trades INT Override the value of the `max_open_trades` configuration setting. @@ -327,14 +340,18 @@ optional arguments: --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). - -e INT, --epochs INT Specify number of epochs (default: 100). - --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...] - Specify which parameters to hyperopt. Space-separated - list. --dmmp, --disable-max-market-positions Disable applying `max_open_trades` during backtest (same as setting `max_open_trades` to a very high number). + --enable-protections, --enableprotections + Enable protections for backtesting.Will slow + backtesting down by a considerable amount, but will + include configured protections + -e INT, --epochs INT Specify number of epochs (default: 100). + --spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...] + Specify which parameters to hyperopt. Space-separated + list. --print-all Print all results, not only the best ones. --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. @@ -353,10 +370,10 @@ optional arguments: class (IHyperOptLoss). Different functions can generate completely different results, since the target for optimization is different. Built-in - Hyperopt-loss-functions are: ShortTradeDurHyperOptLoss, - OnlyProfitHyperOptLoss, SharpeHyperOptLoss, - SharpeHyperOptLossDaily, SortinoHyperOptLoss, - SortinoHyperOptLossDaily. + Hyperopt-loss-functions are: + ShortTradeDurHyperOptLoss, OnlyProfitHyperOptLoss, + SharpeHyperOptLoss, SharpeHyperOptLossDaily, + SortinoHyperOptLoss, SortinoHyperOptLossDaily Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/docs/configuration.md b/docs/configuration.md index 2e8f6555f..b70a85c04 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -91,6 +91,7 @@ Mandatory parameters are marked as **Required**, which means that they are requi | `edge.*` | Please refer to [edge configuration document](edge.md) for detailed explanation. | `experimental.block_bad_exchanges` | Block exchanges known to not work with freqtrade. Leave on default unless you want to test if that exchange works now.
*Defaults to `true`.*
**Datatype:** Boolean | `pairlists` | Define one or more pairlists to be used. [More information below](#pairlists-and-pairlist-handlers).
*Defaults to `StaticPairList`.*
**Datatype:** List of Dicts +| `protections` | Define one or more protections to be used. [More information below](#protections).
**Datatype:** List of Dicts | `telegram.enabled` | Enable the usage of Telegram.
**Datatype:** Boolean | `telegram.token` | Your Telegram bot token. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String | `telegram.chat_id` | Your personal Telegram account id. Only required if `telegram.enabled` is `true`.
**Keep it in secret, do not disclose publicly.**
**Datatype:** String @@ -575,6 +576,7 @@ Assuming both buy and sell are using market orders, a configuration similar to t Obviously, if only one side is using limit orders, different pricing combinations can be used. --8<-- "includes/pairlists.md" +--8<-- "includes/protections.md" ## Switch to Dry-run mode diff --git a/docs/data-download.md b/docs/data-download.md index e9c5c1865..2d77a8a17 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -8,7 +8,7 @@ If no additional parameter is specified, freqtrade will download data for `"1m"` Exchange and pairs will come from `config.json` (if specified using `-c/--config`). Otherwise `--exchange` becomes mandatory. -You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101`). For incremental downloads, the relative approach should be used. +You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used. !!! Tip "Tip: Updating existing data" If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data. diff --git a/docs/developer.md b/docs/developer.md index c253f4460..299f2f77f 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA) where you can ask questions. ## Documentation @@ -94,7 +94,9 @@ Below is an outline of exception inheritance hierarchy: +---+ StrategyError ``` -## Modules +--- + +## Plugins ### Pairlists @@ -119,6 +121,9 @@ The base-class provides an instance of the exchange (`self._exchange`) the pairl self._pairlist_pos = pairlist_pos ``` +!!! Tip + Don't forget to register your pairlist in `constants.py` under the variable `AVAILABLE_PAIRLISTS` - otherwise it will not be selectable. + Now, let's step through the methods which require actions: #### Pairlist configuration @@ -170,6 +175,66 @@ In `VolumePairList`, this implements different methods of sorting, does early va return pairs ``` +### Protections + +Best read the [Protection documentation](configuration.md#protections) to understand protections. +This Guide is directed towards Developers who want to develop a new protection. + +No protection should use datetime directly, but use the provided `date_now` variable for date calculations. This preserves the ability to backtest protections. + +!!! Tip "Writing a new Protection" + Best copy one of the existing Protections to have a good example. + Don't forget to register your protection in `constants.py` under the variable `AVAILABLE_PROTECTIONS` - otherwise it will not be selectable. + +#### Implementation of a new protection + +All Protection implementations must have `IProtection` as parent class. +For that reason, they must implement the following methods: + +* `short_desc()` +* `global_stop()` +* `stop_per_pair()`. + +`global_stop()` and `stop_per_pair()` must return a ProtectionReturn tuple, which consists of: + +* lock pair - boolean +* lock until - datetime - until when should the pair be locked (will be rounded up to the next new candle) +* reason - string, used for logging and storage in the database + +The `until` portion should be calculated using the provided `calculate_lock_end()` method. + +All Protections should use `"stop_duration"` / `"stop_duration_candles"` to define how long a a pair (or all pairs) should be locked. +The content of this is made available as `self._stop_duration` to the each Protection. + +If your protection requires a look-back period, please use `"lookback_period"` / `"lockback_period_candles"` to keep all protections aligned. + +#### Global vs. local stops + +Protections can have 2 different ways to stop trading for a limited : + +* Per pair (local) +* For all Pairs (globally) + +##### Protections - per pair + +Protections that implement the per pair approach must set `has_local_stop=True`. +The method `stop_per_pair()` will be called whenever a trade closed (sell order completed). + +##### Protections - global protection + +These Protections should do their evaluation across all pairs, and consequently will also lock all pairs from trading (called a global PairLock). +Global protection must set `has_global_stop=True` to be evaluated for global stops. +The method `global_stop()` will be called whenever a trade closed (sell order completed). + +##### Protections - calculating lock end time + +Protections should calculate the lock end time based on the last trade it considers. +This avoids re-locking should the lookback-period be longer than the actual lock period. + +The `IProtection` parent class provides a helper method for this in `calculate_lock_end()`. + +--- + ## Implement a new Exchange (WIP) !!! Note @@ -177,6 +242,9 @@ In `VolumePairList`, this implements different methods of sorting, does early va Most exchanges supported by CCXT should work out of the box. +To quickly test the public endpoints of an exchange, add a configuration for your exchange to `test_ccxt_compat.py` and run these tests with `pytest --longrun tests/exchange/test_ccxt_compat.py`. +Completing these tests successfully a good basis point (it's a requirement, actually), however these won't guarantee correct exchange functioning, as this only tests public endpoints, but no private endpoint (like generate order or similar). + ### Stoploss On Exchange Check if the new exchange supports Stoploss on Exchange orders through their API. diff --git a/docs/edge.md b/docs/edge.md index 7442f1927..fd6d2cf7d 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -23,8 +23,8 @@ The Edge Positioning module seeks to improve a strategy's winning probability an We raise the following question[^1]: !!! Question "Which trade is a better option?" - a) A trade with 80% of chance of losing $100 and 20% chance of winning $200
- b) A trade with 100% of chance of losing $30 + a) A trade with 80% of chance of losing 100\$ and 20% chance of winning 200\$
+ b) A trade with 100% of chance of losing 30\$ ???+ Info "Answer" The expected value of *a)* is smaller than the expected value of *b)*.
@@ -34,8 +34,8 @@ We raise the following question[^1]: Another way to look at it is to ask a similar question: !!! Question "Which trade is a better option?" - a) A trade with 80% of chance of winning 100 and 20% chance of losing $200
- b) A trade with 100% of chance of winning $30 + a) A trade with 80% of chance of winning 100\$ and 20% chance of losing 200\$
+ b) A trade with 100% of chance of winning 30\$ Edge positioning tries to answer the hard questions about risk/reward and position size automatically, seeking to minimizes the chances of losing of a given strategy. @@ -82,7 +82,7 @@ Risk Reward Ratio ($R$) is a formula used to measure the expected gains of a giv $$ R = \frac{\text{potential_profit}}{\text{potential_loss}} $$ ???+ Example "Worked example of $R$ calculation" - Let's say that you think that the price of *stonecoin* today is $10.0. You believe that, because they will start mining stonecoin, it will go up to $15.0 tomorrow. There is the risk that the stone is too hard, and the GPUs can't mine it, so the price might go to $0 tomorrow. You are planning to invest $100, which will give you 10 shares (100 / 10). + Let's say that you think that the price of *stonecoin* today is 10.0\$. You believe that, because they will start mining stonecoin, it will go up to 15.0\$ tomorrow. There is the risk that the stone is too hard, and the GPUs can't mine it, so the price might go to 0\$ tomorrow. You are planning to invest 100\$, which will give you 10 shares (100 / 10). Your potential profit is calculated as: @@ -92,9 +92,9 @@ $$ R = \frac{\text{potential_profit}}{\text{potential_loss}} $$ &= 50 \end{aligned}$ - Since the price might go to $0, the $100 dollars invested could turn into 0. + Since the price might go to 0\$, the 100\$ dollars invested could turn into 0. - We do however use a stoploss of 15% - so in the worst case, we'll sell 15% below entry price (or at 8.5$). + We do however use a stoploss of 15% - so in the worst case, we'll sell 15% below entry price (or at 8.5$\). $\begin{aligned} \text{potential_loss} &= (\text{entry_price} - \text{stoploss}) * \frac{\text{investment}}{\text{entry_price}} \\ @@ -109,7 +109,7 @@ $$ R = \frac{\text{potential_profit}}{\text{potential_loss}} $$ &= \frac{50}{15}\\ &= 3.33 \end{aligned}$
- What it effectively means is that the strategy have the potential to make 3.33$ for each $1 invested. + What it effectively means is that the strategy have the potential to make 3.33\$ for each 1\$ invested. On a long horizon, that is, on many trades, we can calculate the risk reward by dividing the strategy' average profit on winning trades by the strategy' average loss on losing trades. We can calculate the average profit, $\mu_{win}$, as follows: @@ -141,7 +141,7 @@ $$E = R * W - L$$ $E = R * W - L = 5 * 0.28 - 0.72 = 0.68$
-The expectancy worked out in the example above means that, on average, this strategy' trades will return 1.68 times the size of its losses. Said another way, the strategy makes $1.68 for every $1 it loses, on average. +The expectancy worked out in the example above means that, on average, this strategy' trades will return 1.68 times the size of its losses. Said another way, the strategy makes 1.68\$ for every 1\$ it loses, on average. This is important for two reasons: First, it may seem obvious, but you know right away that you have a positive return. Second, you now have a number you can compare to other candidate systems to make decisions about which ones you employ. @@ -222,7 +222,7 @@ Edge module has following configuration options: | `stoploss_range_max` | Maximum stoploss.
*Defaults to `-0.10`.*
**Datatype:** Float | `stoploss_range_step` | As an example if this is set to -0.01 then Edge will test the strategy for `[-0.01, -0,02, -0,03 ..., -0.09, -0.10]` ranges.
**Note** than having a smaller step means having a bigger range which could lead to slow calculation.
If you set this parameter to -0.001, you then slow down the Edge calculation by a factor of 10.
*Defaults to `-0.001`.*
**Datatype:** Float | `minimum_winrate` | It filters out pairs which don't have at least minimum_winrate.
This comes handy if you want to be conservative and don't comprise win rate in favour of risk reward ratio.
*Defaults to `0.60`.*
**Datatype:** Float -| `minimum_expectancy` | It filters out pairs which have the expectancy lower than this number.
Having an expectancy of 0.20 means if you put 10$ on a trade you expect a 12$ return.
*Defaults to `0.20`.*
**Datatype:** Float +| `minimum_expectancy` | It filters out pairs which have the expectancy lower than this number.
Having an expectancy of 0.20 means if you put 10\$ on a trade you expect a 12\$ return.
*Defaults to `0.20`.*
**Datatype:** Float | `min_trade_number` | When calculating *W*, *R* and *E* (expectancy) against historical data, you always want to have a minimum number of trades. The more this number is the more Edge is reliable.
Having a win rate of 100% on a single trade doesn't mean anything at all. But having a win rate of 70% over past 100 trades means clearly something.
*Defaults to `10` (it is highly recommended not to decrease this number).*
**Datatype:** Integer | `max_trade_duration_minute` | Edge will filter out trades with long duration. If a trade is profitable after 1 month, it is hard to evaluate the strategy based on it. But if most of trades are profitable and they have maximum duration of 30 minutes, then it is clearly a good sign.
**NOTICE:** While configuring this value, you should take into consideration your timeframe. As an example filtering out trades having duration less than one day for a strategy which has 4h interval does not make sense. Default value is set assuming your strategy interval is relatively small (1m or 5m, etc.).
*Defaults to `1440` (one day).*
**Datatype:** Integer | `remove_pumps` | Edge will remove sudden pumps in a given market while going through historical data. However, given that pumps happen very often in crypto markets, we recommend you keep this off.
*Defaults to `false`.*
**Datatype:** Boolean diff --git a/docs/faq.md b/docs/faq.md index aa33218fb..5742f512a 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -2,30 +2,30 @@ ## Beginner Tips & Tricks -* When you work with your strategy & hyperopt file you should use a proper code editor like vscode or Pycharm. A good code editor will provide syntax highlighting as well as line numbers, making it easy to find syntax errors (most likely, pointed out by Freqtrade during startup). +* When you work with your strategy & hyperopt file you should use a proper code editor like VSCode or PyCharm. A good code editor will provide syntax highlighting as well as line numbers, making it easy to find syntax errors (most likely pointed out by Freqtrade during startup). ## Freqtrade common issues ### The bot does not start -Running the bot with `freqtrade trade --config config.json` does show the output `freqtrade: command not found`. +Running the bot with `freqtrade trade --config config.json` shows the output `freqtrade: command not found`. -This could have the following reasons: +This could be caused by the following reasons: -* The virtual environment is not active - * run `source .env/bin/activate` to activate the virtual environment +* The virtual environment is not active. + * Run `source .env/bin/activate` to activate the virtual environment. * The installation did not work correctly. * Please check the [Installation documentation](installation.md). -### I have waited 5 minutes, why hasn't the bot made any trades yet?! +### I have waited 5 minutes, why hasn't the bot made any trades yet? * Depending on the buy strategy, the amount of whitelisted coins, the -situation of the market etc, it can take up to hours to find good entry +situation of the market etc, it can take up to hours to find a good entry position for a trade. Be patient! -* Or it may because of a configuration error? Best check the logs, it's usually telling you if the bot is simply not getting buy signals (only heartbeat messages), or if there is something wrong (errors / exceptions in the log). +* It may be because of a configuration error. It's best to check the logs, they usually tell you if the bot is simply not getting buy signals (only heartbeat messages), or if there is something wrong (errors / exceptions in the log). -### I have made 12 trades already, why is my total profit negative?! +### I have made 12 trades already, why is my total profit negative? I understand your disappointment but unfortunately 12 trades is just not enough to say anything. If you run backtesting, you can see that our @@ -36,11 +36,9 @@ of course constantly aim to improve the bot but it will _always_ be a gamble, which should leave you with modest wins on monthly basis but you can't say much from few trades. -### I’d like to change the stake amount. Can I just stop the bot with /stop and then change the config.json and run it again? +### I’d like to make changes to the config. Can I do that without having to kill the bot? -Not quite. Trades are persisted to a database but the configuration is -currently only read when the bot is killed and restarted. `/stop` more -like pauses. You can stop your bot, adjust settings and start it again. +Yes. You can edit your config, use the `/stop` command in Telegram, followed by `/reload_config` and the bot will run with the new config. ### I want to improve the bot with a new strategy @@ -49,7 +47,7 @@ the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-c ### Is there a setting to only SELL the coins being held and not perform anymore BUYS? -You can use the `/forcesell all` command from Telegram. +You can use the `/stopbuy` command in Telegram to prevent future buys, followed by `/forcesell all` (sell all open trades). ### I want to run multiple bots on the same machine @@ -59,7 +57,7 @@ Please look at the [advanced setup documentation Page](advanced-setup.md#running This message is just a warning that the latest candles had missing candles in them. Depending on the exchange, this can indicate that the pair didn't have a trade for the timeframe you are using - and the exchange does only return candles with volume. -On low volume pairs, this is a rather common occurance. +On low volume pairs, this is a rather common occurrence. If this happens for all pairs in the pairlist, this might indicate a recent exchange downtime. Please check your exchange's public channels for details. @@ -73,7 +71,7 @@ Read [the Bittrex section about restricted markets](exchanges.md#restricted-mark ### I'm getting the "Exchange Bittrex does not support market orders." message and cannot run my strategy -As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Probably your strategy was written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex). +As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex). To fix it for Bittrex, redefine order types in the strategy to use "limit" instead of "market": @@ -85,7 +83,7 @@ To fix it for Bittrex, redefine order types in the strategy to use "limit" inste } ``` -Same fix should be done in the configuration file, if order types are defined in your custom config rather than in the strategy. +The same fix should be applied in the configuration file, if order types are defined in your custom config rather than in the strategy. ### How do I search the bot logs for something? @@ -127,10 +125,10 @@ On Windows, the `--logfile` option is also supported by Freqtrade and you can us ## Hyperopt module -### How many epoch do I need to get a good Hyperopt result? +### How many epochs do I need to get a good Hyperopt result? Per default Hyperopt called without the `-e`/`--epochs` command line option will only -run 100 epochs, means 100 evals of your triggers, guards, ... Too few +run 100 epochs, means 100 evaluations of your triggers, guards, ... Too few to find a great result (unless if you are very lucky), so you probably have to run it for 10.000 or more. But it will take an eternity to compute. @@ -140,32 +138,32 @@ Since hyperopt uses Bayesian search, running for too many epochs may not produce It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epochs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going. ```bash -freqtrade hyperopt --hyperop SampleHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy SampleStrategy -e 1000 +freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy SampleStrategy -e 1000 ``` ### Why does it take a long time to run hyperopt? -* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. +* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers: This answer was written during the release 0.15.1, when we had: -- 8 triggers -- 9 guards: let's say we evaluate even 10 values from each -- 1 stoploss calculation: let's say we want 10 values from that too to be evaluated +* 8 triggers +* 9 guards: let's say we evaluate even 10 values from each +* 1 stoploss calculation: let's say we want 10 values from that too to be evaluated The following calculation is still very rough and not very precise but it will give the idea. With only these triggers and guards there is -already 8\*10^9\*10 evaluations. A roughly total of 80 billion evals. -Did you run 100 000 evals? Congrats, you've done roughly 1 / 100 000 th +already 8\*10^9\*10 evaluations. A roughly total of 80 billion evaluations. +Did you run 100 000 evaluations? Congrats, you've done roughly 1 / 100 000 th of the search space, assuming that the bot never tests the same parameters more than once. * The time it takes to run 1000 hyperopt epochs depends on things like: The available cpu, hard-disk, ram, timeframe, timerange, indicator settings, indicator count, amount of coins that hyperopt test strategies on and the resulting trade count - which can be 650 trades in a year or 10.0000 trades depending if the strategy aims for big profits by trading rarely or for many low profit trades. Example: 4% profit 650 times vs 0,3% profit a trade 10.000 times in a year. If we assume you set the --timerange to 365 days. -Example: +Example: `freqtrade --config config.json --strategy SampleStrategy --hyperopt SampleHyperopt -e 1000 --timerange 20190601-20200601` ## Edge module diff --git a/docs/hyperopt.md b/docs/hyperopt.md index fc7a0dd93..f88d9cd4f 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -64,9 +64,9 @@ Depending on the space you want to optimize, only some of the below are required Optional in hyperopt - can also be loaded from a strategy (recommended): -* copy `populate_indicators` from your strategy - otherwise default-strategy will be used -* copy `populate_buy_trend` from your strategy - otherwise default-strategy will be used -* copy `populate_sell_trend` from your strategy - otherwise default-strategy will be used +* `populate_indicators` - fallback to create indicators +* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy +* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy !!! Note You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods. @@ -104,7 +104,7 @@ This command will create a new hyperopt file from a template, allowing you to ge There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing: * Inside `indicator_space()` - the parameters hyperopt shall be optimizing. -* Inside `populate_buy_trend()` - applying the parameters. +* Within `buy_strategy_generator()` - populate the nested `populate_buy_trend()` to apply the parameters. There you have two different types of indicators: 1. `guards` and 2. `triggers`. @@ -128,7 +128,7 @@ Similar to the buy-signal above, sell-signals can also be optimized. Place the corresponding settings into the following methods * Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. -* Inside `populate_sell_trend()` - applying the parameters. +* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters. The configuration and rules are the same than for buy signals. To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. @@ -173,6 +173,11 @@ one we call `trigger` and use it to decide which buy trigger we want to use. So let's write the buy strategy using these values: ```python + @staticmethod + def buy_strategy_generator(params: Dict[str, Any]) -> Callable: + """ + Define the buy strategy parameters to be used by Hyperopt. + """ def populate_buy_trend(dataframe: DataFrame) -> DataFrame: conditions = [] # GUARDS AND TRENDS diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 5bb02470d..732dfa5bb 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -15,6 +15,7 @@ Inactive markets are always removed from the resulting pairlist. Explicitly blac * [`StaticPairList`](#static-pair-list) (default, if not configured differently) * [`VolumePairList`](#volume-pair-list) * [`AgeFilter`](#agefilter) +* [`PerformanceFilter`](#performancefilter) * [`PrecisionFilter`](#precisionfilter) * [`PriceFilter`](#pricefilter) * [`ShuffleFilter`](#shufflefilter) @@ -64,6 +65,9 @@ The `refresh_period` setting allows to define the period (in seconds), at which }], ``` +!!! Note + `VolumePairList` does not support backtesting mode. + #### AgeFilter Removes pairs that have been listed on the exchange for less than `min_days_listed` days (defaults to `10`). @@ -74,6 +78,18 @@ be caught out buying before the pair has finished dropping in price. This filter allows freqtrade to ignore pairs until they have been listed for at least `min_days_listed` days. +#### PerformanceFilter + +Sorts pairs by past trade performance, as follows: +1. Positive performance. +2. No closed trades yet. +3. Negative performance. + +Trade count is used as a tie breaker. + +!!! Note + `PerformanceFilter` does not support backtesting mode. + #### PrecisionFilter Filters low-value coins which would not allow setting stoplosses. diff --git a/docs/includes/protections.md b/docs/includes/protections.md new file mode 100644 index 000000000..87db17fd8 --- /dev/null +++ b/docs/includes/protections.md @@ -0,0 +1,169 @@ +## Protections + +!!! Warning "Beta feature" + This feature is still in it's testing phase. Should you notice something you think is wrong please let us know via Discord, Slack or via Github Issue. + +Protections will protect your strategy from unexpected events and market conditions by temporarily stop trading for either one pair, or for all pairs. +All protection end times are rounded up to the next candle to avoid sudden, unexpected intra-candle buys. + +!!! Note + Not all Protections will work for all strategies, and parameters will need to be tuned for your strategy to improve performance. + +!!! Tip + Each Protection can be configured multiple times with different parameters, to allow different levels of protection (short-term / long-term). + +!!! Note "Backtesting" + Protections are supported by backtesting and hyperopt, but must be explicitly enabled by using the `--enable-protections` flag. + +### Available Protections + +* [`StoplossGuard`](#stoploss-guard) Stop trading if a certain amount of stoploss occurred within a certain time window. +* [`MaxDrawdown`](#maxdrawdown) Stop trading if max-drawdown is reached. +* [`LowProfitPairs`](#low-profit-pairs) Lock pairs with low profits +* [`CooldownPeriod`](#cooldown-period) Don't enter a trade right after selling a trade. + +### Common settings to all Protections + +| Parameter| Description | +|------------|-------------| +| `method` | Protection name to use.
**Datatype:** String, selected from [available Protections](#available-protections) +| `stop_duration_candles` | For how many candles should the lock be set?
**Datatype:** Positive integer (in candles) +| `stop_duration` | how many minutes should protections be locked.
Cannot be used together with `stop_duration_candles`.
**Datatype:** Float (in minutes) +| `lookback_period_candles` | Only trades that completed within the last `lookback_period_candles` candles will be considered. This setting may be ignored by some Protections.
**Datatype:** Positive integer (in candles). +| `lookback_period` | Only trades that completed after `current_time - lookback_period` will be considered.
Cannot be used together with `lookback_period_candles`.
This setting may be ignored by some Protections.
**Datatype:** Float (in minutes) +| `trade_limit` | Number of trades required at minimum (not used by all Protections).
**Datatype:** Positive integer + +!!! Note "Durations" + Durations (`stop_duration*` and `lookback_period*` can be defined in either minutes or candles). + For more flexibility when testing different timeframes, all below examples will use the "candle" definition. + +#### Stoploss Guard + +`StoplossGuard` selects all trades within `lookback_period`, and determines if the amount of trades that resulted in stoploss are above `trade_limit` - in which case trading will stop for `stop_duration`. +This applies across all pairs, unless `only_per_pair` is set to true, which will then only look at one pair at a time. + +The below example stops trading for all pairs for 4 candles after the last trade if the bot hit stoploss 4 times within the last 24 candles. + +```json +"protections": [ + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 4, + "only_per_pair": false + } +], +``` + +!!! Note + `StoplossGuard` considers all trades with the results `"stop_loss"` and `"trailing_stop_loss"` if the resulting profit was negative. + `trade_limit` and `lookback_period` will need to be tuned for your strategy. + +#### MaxDrawdown + +`MaxDrawdown` uses all trades within `lookback_period` (in minutes) to determine the maximum drawdown. If the drawdown is below `max_allowed_drawdown`, trading will stop for `stop_duration` (in minutes) after the last trade - assuming that the bot needs some time to let markets recover. + +The below sample stops trading for 12 candles if max-drawdown is > 20% considering all trades within the last 48 candles. + +```json +"protections": [ + { + "method": "MaxDrawdown", + "lookback_period_candles": 48, + "trade_limit": 20, + "stop_duration_candles": 12, + "max_allowed_drawdown": 0.2 + }, +], + +``` + +#### Low Profit Pairs + +`LowProfitPairs` uses all trades for a pair within `lookback_period` (in minutes) to determine the overall profit ratio. +If that ratio is below `required_profit`, that pair will be locked for `stop_duration` (in minutes). + +The below example will stop trading a pair for 60 minutes if the pair does not have a required profit of 2% (and a minimum of 2 trades) within the last 6 candles. + +```json +"protections": [ + { + "method": "LowProfitPairs", + "lookback_period_candles": 6, + "trade_limit": 2, + "stop_duration": 60, + "required_profit": 0.02 + } +], +``` + +#### Cooldown Period + +`CooldownPeriod` locks a pair for `stop_duration` (in minutes) after selling, avoiding a re-entry for this pair for `stop_duration` minutes. + +The below example will stop trading a pair for 2 candles after closing a trade, allowing this pair to "cool down". + +```json +"protections": [ + { + "method": "CooldownPeriod", + "stop_duration_candles": 2 + } +], +``` + +!!! Note + This Protection applies only at pair-level, and will never lock all pairs globally. + This Protection does not consider `lookback_period` as it only looks at the latest trade. + +### Full example of Protections + +All protections can be combined at will, also with different parameters, creating a increasing wall for under-performing pairs. +All protections are evaluated in the sequence they are defined. + +The below example assumes a timeframe of 1 hour: + +* Locks each pair after selling for an additional 5 candles (`CooldownPeriod`), giving other pairs a chance to get filled. +* Stops trading for 4 hours (`4 * 1h candles`) if the last 2 days (`48 * 1h candles`) had 20 trades, which caused a max-drawdown of more than 20%. (`MaxDrawdown`). +* Stops trading if more than 4 stoploss occur for all pairs within a 1 day (`24 * 1h candles`) limit (`StoplossGuard`). +* Locks all pairs that had 4 Trades within the last 6 hours (`6 * 1h candles`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`). +* Locks all pairs for 2 candles that had a profit of below 0.01 (<1%) within the last 24h (`24 * 1h candles`), a minimum of 4 trades. + +```json +"timeframe": "1h", +"protections": [ + { + "method": "CooldownPeriod", + "stop_duration_candles": 5 + }, + { + "method": "MaxDrawdown", + "lookback_period_candles": 48, + "trade_limit": 20, + "stop_duration_candles": 4, + "max_allowed_drawdown": 0.2 + }, + { + "method": "StoplossGuard", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 2, + "only_per_pair": false + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 6, + "trade_limit": 2, + "stop_duration_candles": 60, + "required_profit": 0.02 + }, + { + "method": "LowProfitPairs", + "lookback_period_candles": 24, + "trade_limit": 4, + "stop_duration_candles": 2, + "required_profit": 0.01 + } + ], +``` diff --git a/docs/index.md b/docs/index.md index f63aeb6b8..38e040d7a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,7 +14,7 @@ ## Introduction -Freqtrade is a crypto-currency algorithmic trading software developed in python (3.6+) and supported on Windows, macOS and Linux. +Freqtrade is a crypto-currency algorithmic trading software developed in python (3.7+) and supported on Windows, macOS and Linux. !!! Danger "DISCLAIMER" This software is for educational purposes only. Do not risk money which you are afraid to lose. USE THE SOFTWARE AT YOUR OWN RISK. THE AUTHORS AND ALL AFFILIATES ASSUME NO RESPONSIBILITY FOR YOUR TRADING RESULTS. @@ -51,7 +51,7 @@ To run this bot we recommend you a linux cloud instance with a minimum of: Alternatively -- Python 3.6.x +- Python 3.7+ - pip (pip3) - git - TA-Lib @@ -65,7 +65,7 @@ For any questions not covered by the documentation or for further information ab Please check out our [discord server](https://discord.gg/MA9v74M). -You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-jaut7r4m-Y17k4x5mcQES9a9swKuxbg). +You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-k9o2v5ut-jX8Mc4CwNM8CDc2Dyg96YA). ## Ready to try? diff --git a/docs/installation.md b/docs/installation.md index 9b15c9685..be98c45a8 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -10,7 +10,7 @@ Please consider using the prebuilt [docker images](docker.md) to get started qui Click each one for install guide: -* [Python >= 3.6.x](http://docs.python-guide.org/en/latest/starting/installation/) +* [Python >= 3.7.x](http://docs.python-guide.org/en/latest/starting/installation/) * [pip](https://pip.pypa.io/en/stable/installing/) * [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) * [virtualenv](https://virtualenv.pypa.io/en/stable/installation.html) (Recommended) @@ -34,7 +34,7 @@ The easiest way to install and run Freqtrade is to clone the bot Github reposito When cloning the repository the default working branch has the name `develop`. This branch contains all last features (can be considered as relatively stable, thanks to automated tests). The `stable` branch contains the code of the last release (done usually once per month on an approximately one week old snapshot of the `develop` branch to prevent packaging bugs, so potentially it's more stable). !!! Note - Python3.6 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. + Python3.7 or higher and the corresponding `pip` are assumed to be available. The install-script will warn you and stop if that's not the case. `git` is also needed to clone the Freqtrade repository. This can be achieved with the following commands: @@ -63,7 +63,7 @@ usage: ** --install ** With this option, the script will install the bot and most dependencies: -You will need to have git and python3.6+ installed beforehand for this to work. +You will need to have git and python3.7+ installed beforehand for this to work. * Mandatory software as: `ta-lib` * Setup your virtualenv under `.env/` @@ -90,13 +90,13 @@ Each time you open a new terminal, you must run `source .env/bin/activate`. ## Custom Installation -We've included/collected install instructions for Ubuntu 16.04, MacOS, and Windows. These are guidelines and your success may vary with other distros. +We've included/collected install instructions for Ubuntu, MacOS, and Windows. These are guidelines and your success may vary with other distros. OS Specific steps are listed first, the [Common](#common) section below is necessary for all systems. !!! Note - Python3.6 or higher and the corresponding pip are assumed to be available. + Python3.7 or higher and the corresponding pip are assumed to be available. -=== "Ubuntu 16.04" +=== "Ubuntu/Debian" #### Install necessary dependencies ```bash @@ -105,13 +105,17 @@ OS Specific steps are listed first, the [Common](#common) section below is neces ``` === "RaspberryPi/Raspbian" - The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/) from at least September 2019. + The following assumes the latest [Raspbian Buster lite image](https://www.raspberrypi.org/downloads/raspbian/). This image comes with python3.7 preinstalled, making it easy to get freqtrade up and running. Tested using a Raspberry Pi 3 with the Raspbian Buster lite image, all updates applied. + ``` bash - sudo apt-get install python3-venv libatlas-base-dev + sudo apt-get install python3-venv libatlas-base-dev cmake + # Use pywheels.org to speed up installation + sudo echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > tee /etc/pip.conf + git clone https://github.com/freqtrade/freqtrade.git cd freqtrade @@ -120,6 +124,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces !!! Note "Installation duration" Depending on your internet speed and the Raspberry Pi version, installation can take multiple hours to complete. + Due to this, we recommend to use the prebuild docker-image for Raspberry, by following the [Docker quickstart documentation](docker_quickstart.md) !!! Note The above does not install hyperopt dependencies. To install these, please use `python3 -m pip install -e .[hyperopt]`. diff --git a/docs/partials/header.html b/docs/partials/header.html index 32202bccc..f5243225b 100644 --- a/docs/partials/header.html +++ b/docs/partials/header.html @@ -1,54 +1,51 @@ +{#- +This file was automatically generated - do not edit +-#} +{% set site_url = config.site_url | d(nav.homepage.url, true) | url %} +{% if not config.use_directory_urls and site_url[0] == site_url[-1] == "." %} +{% set site_url = site_url ~ "/index.html" %} +{% endif %}
- + + +
diff --git a/docs/plotting.md b/docs/plotting.md index 09eb6ddb5..ed682e44b 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -168,6 +168,7 @@ Additional features when using plot_config include: * Specify colors per indicator * Specify additional subplots +* Specify indicator pairs to fill area in between The sample plot configuration below specifies fixed colors for the indicators. Otherwise consecutive plots may produce different colorschemes each time, making comparisons difficult. It also allows multiple subplots to display both MACD and RSI at the same time. @@ -183,23 +184,33 @@ Sample configuration with inline comments explaining the process: 'ema50': {'color': '#CCCCCC'}, # By omitting color, a random color is selected. 'sar': {}, + # fill area between senkou_a and senkou_b + 'senkou_a': { + 'color': 'green', #optional + 'fill_to': 'senkou_b', + 'fill_label': 'Ichimoku Cloud' #optional, + 'fill_color': 'rgba(255,76,46,0.2)', #optional + }, + # plot senkou_b, too. Not only the area to it. + 'senkou_b': {} }, 'subplots': { # Create subplot MACD "MACD": { - 'macd': {'color': 'blue'}, - 'macdsignal': {'color': 'orange'}, + 'macd': {'color': 'blue', 'fill_to': 'macdhist'}, + 'macdsignal': {'color': 'orange'} }, # Additional subplot RSI "RSI": { - 'rsi': {'color': 'red'}, + 'rsi': {'color': 'red'} } } } -``` +``` !!! Note - The above configuration assumes that `ema10`, `ema50`, `macd`, `macdsignal` and `rsi` are columns in the DataFrame created by the strategy. + The above configuration assumes that `ema10`, `ema50`, `senkou_a`, `senkou_b`, + `macd`, `macdsignal`, `macdhist` and `rsi` are columns in the DataFrame created by the strategy. ## Plot profit diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 000000000..1f785bbaa --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,3 @@ +# Plugins +--8<-- "includes/pairlists.md" +--8<-- "includes/protections.md" diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 87bc6dfdd..2db336f4a 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,3 @@ -mkdocs-material==6.1.6 +mkdocs-material==6.2.3 mdx_truly_sane_lists==1.2 -pymdown-extensions==8.0.1 +pymdown-extensions==8.1 diff --git a/docs/rest-api.md b/docs/rest-api.md index 7726ab875..9bb35ce91 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -127,6 +127,7 @@ python3 scripts/rest_client.py --config rest_config.json [optional par | `performance` | Show performance of each finished trade grouped by pair. | `balance` | Show account balance per currency. | `daily ` | Shows profit or loss per day, over the last n days (n defaults to 7). +| `stats` | Display a summary of profit / loss reasons as well as average holding times. | `whitelist` | Show the current whitelist. | `blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist. | `edge` | Show validated pairs by Edge if it is enabled. @@ -229,6 +230,9 @@ show_config start Start the bot if it's in the stopped state. +stats + Return the stats report (durations, sell-reasons). + status Get the status of open trades. diff --git a/docs/strategy-customization.md b/docs/strategy-customization.md index db007985f..ab64d3a67 100644 --- a/docs/strategy-customization.md +++ b/docs/strategy-customization.md @@ -147,7 +147,7 @@ Let's try to backtest 1 month (January 2019) of 5m candles using an example stra freqtrade backtesting --timerange 20190101-20190201 --timeframe 5m ``` -Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2019-12-31 15:30:00. +Assuming `startup_candle_count` is set to 100, backtesting knows it needs 100 candles to generate valid buy signals. It will load data from `20190101 - (100 * 5m)` - which is ~2018-12-31 15:30:00. If this data is available, indicators will be calculated with this extended timerange. The instable startup period (up to 2019-01-01 00:00:00) will then be removed before starting backtesting. !!! Note diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 09cf21223..40481684d 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -87,6 +87,41 @@ Example configuration showing the different settings: }, ``` +## Create a custom keyboard (command shortcut buttons) + +Telegram allows us to create a custom keyboard with buttons for commands. +The default custom keyboard looks like this. + +```python +[ + ["/daily", "/profit", "/balance"], # row 1, 3 commands + ["/status", "/status table", "/performance"], # row 2, 3 commands + ["/count", "/start", "/stop", "/help"] # row 3, 4 commands +] +``` + +### Usage + +You can create your own keyboard in `config.json`: + +``` json +"telegram": { + "enabled": true, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id", + "keyboard": [ + ["/daily", "/stats", "/balance", "/profit"], + ["/status table", "/performance"], + ["/reload_config", "/count", "/logs"] + ] + }, +``` + +!!! Note "Supported Commands" + Only the following commands are allowed. Command arguments are not supported! + + `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopbuy`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version` + ## Telegram commands Per default, the Telegram bot shows predefined commands. Some commands @@ -106,6 +141,7 @@ official commands. You can ask at any moment for help with `/help`. | `/trades [limit]` | List all recently closed trades in a table format. | `/delete ` | Delete a specific trade from the Database. Tries to close open orders. Requires manual handling of this trade on the exchange. | `/count` | Displays number of trades used and available +| `/locks` | Show currently locked pairs. | `/profit` | Display a summary of your profit/loss from close trades and some stats about your performance | `/forcesell ` | Instantly sells the given trade (Ignoring `minimum_roi`). | `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`). @@ -113,6 +149,7 @@ official commands. You can ask at any moment for help with `/help`. | `/performance` | Show performance of each finished trade grouped by pair | `/balance` | Show account balance per currency | `/daily ` | Shows profit or loss per day, over the last n days (n defaults to 7) +| `/stats` | Shows Wins / losses by Sell reason as well as Avg. holding durations for buys and sells | `/whitelist` | Show the current whitelist | `/blacklist [pair]` | Show the current blacklist, or adds a pair to the blacklist. | `/edge` | Show validated pairs by Edge if it is enabled. @@ -207,7 +244,7 @@ Return a summary of your profit/loss and performance. Note that for this to work, `forcebuy_enable` needs to be set to true. -[More details](configuration.md/#understand-forcebuy_enable) +[More details](configuration.md#understand-forcebuy_enable) ### /performance diff --git a/docs/updating.md b/docs/updating.md new file mode 100644 index 000000000..b23ce32dc --- /dev/null +++ b/docs/updating.md @@ -0,0 +1,31 @@ +# How to update + +To update your freqtrade installation, please use one of the below methods, corresponding to your installation method. + +## docker-compose + +!!! Note "Legacy installations using the `master` image" + We're switching from master to stable for the release Images - please adjust your docker-file and replace `freqtradeorg/freqtrade:master` with `freqtradeorg/freqtrade:stable` + +``` bash +docker-compose pull +docker-compose up -d +``` + +## Installation via setup script + +``` bash +./setup.sh --update +``` + +!!! Note + Make sure to run this command with your virtual environment disabled! + +## Plain native installation + +Please ensure that you're also updating dependencies - otherwise things might break without you noticing. + +``` bash +git pull +pip install -U -r requirements.txt +``` diff --git a/environment.yml b/environment.yml index 86ea03519..746c4b912 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: - conda-forge dependencies: # Required for app - - python>=3.6 + - python>=3.7 - pip - wheel - numpy diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 3054bc4a1..170f95015 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2020.11' +__version__ = '2020.12' if __version__ == 'develop': diff --git a/freqtrade/__main__.py b/freqtrade/__main__.py index 881a2f562..ab4c7a110 100644 --- a/freqtrade/__main__.py +++ b/freqtrade/__main__.py @@ -3,7 +3,7 @@ __main__.py for Freqtrade To launch Freqtrade as a module -> python -m freqtrade (with Python >= 3.6) +> python -m freqtrade (with Python >= 3.7) """ from freqtrade import main diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index aa58ff585..a6c8a245f 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -20,11 +20,13 @@ ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv", "max_open_trades", "stake_amount", "fee"] ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions", + "enable_protections", "strategy_list", "export", "exportfilename"] ARGS_HYPEROPT = ARGS_COMMON_OPTIMIZE + ["hyperopt", "hyperopt_path", - "position_stacking", "epochs", "spaces", - "use_max_market_positions", "print_all", + "position_stacking", "use_max_market_positions", + "enable_protections", + "epochs", "spaces", "print_all", "print_colorized", "print_json", "hyperopt_jobs", "hyperopt_random_state", "hyperopt_min_trades", "hyperopt_loss"] @@ -42,7 +44,8 @@ ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"] ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one_column", "print_csv", "base_currencies", "quote_currencies", "list_pairs_all"] -ARGS_TEST_PAIRLIST = ["config", "quote_currencies", "print_one_column", "list_pairs_print_json"] +ARGS_TEST_PAIRLIST = ["verbosity", "config", "quote_currencies", "print_one_column", + "list_pairs_print_json"] ARGS_CREATE_USERDIR = ["user_data_dir", "reset"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 619a300ae..668b4abf5 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -144,6 +144,14 @@ AVAILABLE_CLI_OPTIONS = { action='store_false', default=True, ), + "enable_protections": Arg( + '--enable-protections', '--enableprotections', + help='Enable protections for backtesting.' + 'Will slow backtesting down by a considerable amount, but will include ' + 'configured protections', + action='store_true', + default=False, + ), "strategy_list": Arg( '--strategy-list', help='Provide a space-separated list of strategies to backtest. ' diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index e4ee80ca5..0661cd03c 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -15,7 +15,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: """ Test Pairlist configuration """ - from freqtrade.pairlist.pairlistmanager import PairListManager + from freqtrade.plugins.pairlistmanager import PairListManager config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE) exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config, validate=False) diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index ab21bc686..b8829b80f 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -74,6 +74,7 @@ def validate_config_consistency(conf: Dict[str, Any]) -> None: _validate_trailing_stoploss(conf) _validate_edge(conf) _validate_whitelist(conf) + _validate_protections(conf) _validate_unlimited_amount(conf) # validate configuration before returning @@ -155,3 +156,22 @@ def _validate_whitelist(conf: Dict[str, Any]) -> None: if (pl.get('method') == 'StaticPairList' and not conf.get('exchange', {}).get('pair_whitelist')): raise OperationalException("StaticPairList requires pair_whitelist to be set.") + + +def _validate_protections(conf: Dict[str, Any]) -> None: + """ + Validate protection configuration validity + """ + + for prot in conf.get('protections', []): + if ('stop_duration' in prot and 'stop_duration_candles' in prot): + raise OperationalException( + "Protections must specify either `stop_duration` or `stop_duration_candles`.\n" + f"Please fix the protection {prot.get('method')}" + ) + + if ('lookback_period' in prot and 'lookback_period_candles' in prot): + raise OperationalException( + "Protections must specify either `lookback_period` or `lookback_period_candles`.\n" + f"Please fix the protection {prot.get('method')}" + ) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 1ca3187fb..7bf3e6bf2 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -211,6 +211,9 @@ class Configuration: self._args_to_config(config, argname='position_stacking', logstring='Parameter --enable-position-stacking detected ...') + self._args_to_config( + config, argname='enable_protections', + logstring='Parameter --enable-protections detected, enabling Protections. ...') # Setting max_open_trades to infinite if -1 if config.get('max_open_trades') == -1: config['max_open_trades'] = float('inf') diff --git a/freqtrade/configuration/deprecated_settings.py b/freqtrade/configuration/deprecated_settings.py index 03ed41ab8..6b2a20c8c 100644 --- a/freqtrade/configuration/deprecated_settings.py +++ b/freqtrade/configuration/deprecated_settings.py @@ -26,6 +26,24 @@ def check_conflicting_settings(config: Dict[str, Any], ) +def process_removed_setting(config: Dict[str, Any], + section1: str, name1: str, + section2: str, name2: str) -> None: + """ + :param section1: Removed section + :param name1: Removed setting name + :param section2: new section for this key + :param name2: new setting name + """ + section1_config = config.get(section1, {}) + if name1 in section1_config: + raise OperationalException( + f"Setting `{section1}.{name1}` has been moved to `{section2}.{name2}. " + f"Please delete it from your configuration and use the `{section2}.{name2}` " + "setting instead." + ) + + def process_deprecated_setting(config: Dict[str, Any], section1: str, name1: str, section2: str, name2: str) -> None: @@ -44,19 +62,18 @@ def process_deprecated_setting(config: Dict[str, Any], def process_temporary_deprecated_settings(config: Dict[str, Any]) -> None: - check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal', - 'experimental', 'use_sell_signal') - check_conflicting_settings(config, 'ask_strategy', 'sell_profit_only', - 'experimental', 'sell_profit_only') - check_conflicting_settings(config, 'ask_strategy', 'ignore_roi_if_buy_signal', - 'experimental', 'ignore_roi_if_buy_signal') + # Kept for future deprecated / moved settings + # check_conflicting_settings(config, 'ask_strategy', 'use_sell_signal', + # 'experimental', 'use_sell_signal') + # process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal', + # 'experimental', 'use_sell_signal') - process_deprecated_setting(config, 'ask_strategy', 'use_sell_signal', - 'experimental', 'use_sell_signal') - process_deprecated_setting(config, 'ask_strategy', 'sell_profit_only', - 'experimental', 'sell_profit_only') - process_deprecated_setting(config, 'ask_strategy', 'ignore_roi_if_buy_signal', - 'experimental', 'ignore_roi_if_buy_signal') + process_removed_setting(config, 'experimental', 'use_sell_signal', + 'ask_strategy', 'use_sell_signal') + process_removed_setting(config, 'experimental', 'sell_profit_only', + 'ask_strategy', 'sell_profit_only') + process_removed_setting(config, 'experimental', 'ignore_roi_if_buy_signal', + 'ask_strategy', 'ignore_roi_if_buy_signal') if (config.get('edge', {}).get('enabled', False) and 'capital_available_percentage' in config.get('edge', {})): diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 2022556d2..e7d7e80f6 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,8 +24,10 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', - 'AgeFilter', 'PrecisionFilter', 'PriceFilter', - 'RangeStabilityFilter', 'ShuffleFilter', 'SpreadFilter'] + 'AgeFilter', 'PerformanceFilter', 'PrecisionFilter', + 'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter', + 'SpreadFilter'] +AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard'] AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5'] DRY_RUN_WALLET = 1000 DATETIME_PRINT_FORMAT = '%Y-%m-%d %H:%M:%S' @@ -182,9 +184,6 @@ CONF_SCHEMA = { 'experimental': { 'type': 'object', 'properties': { - 'use_sell_signal': {'type': 'boolean'}, - 'sell_profit_only': {'type': 'boolean'}, - 'ignore_roi_if_buy_signal': {'type': 'boolean'}, 'block_bad_exchanges': {'type': 'boolean'} } }, @@ -194,7 +193,21 @@ CONF_SCHEMA = { 'type': 'object', 'properties': { 'method': {'type': 'string', 'enum': AVAILABLE_PAIRLISTS}, - 'config': {'type': 'object'} + }, + 'required': ['method'], + } + }, + 'protections': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'method': {'type': 'string', 'enum': AVAILABLE_PROTECTIONS}, + 'stop_duration': {'type': 'number', 'minimum': 0.0}, + 'stop_duration_candles': {'type': 'number', 'minimum': 0}, + 'trade_limit': {'type': 'number', 'minimum': 1}, + 'lookback_period': {'type': 'number', 'minimum': 1}, + 'lookback_period_candles': {'type': 'number', 'minimum': 1}, }, 'required': ['method'], } diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 5b58d7a95..15ba7b9f6 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -6,6 +6,7 @@ from freqtrade.exchange.exchange import Exchange from freqtrade.exchange.bibox import Bibox from freqtrade.exchange.binance import Binance from freqtrade.exchange.bittrex import Bittrex +from freqtrade.exchange.bybit import Bybit from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges, get_exchange_bad_reason, is_exchange_bad, is_exchange_known_ccxt, is_exchange_officially_supported, diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 099f282a2..26ec30a8a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -18,6 +18,7 @@ class Binance(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, "order_time_in_force": ['gtc', 'fok', 'ioc'], + "ohlcv_candle_limit": 1000, "trades_pagination": "id", "trades_pagination_arg": "fromId", "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000], diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py new file mode 100644 index 000000000..4a44bb42d --- /dev/null +++ b/freqtrade/exchange/bybit.py @@ -0,0 +1,24 @@ +""" Bybit exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Bybit(Exchange): + """ + Bybit exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + # fetchCurrencies API point requires authentication for Bybit, + _ft_has: Dict = { + "ohlcv_candle_limit": 200, + } diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 611ce4abd..6f495e605 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -658,7 +658,8 @@ class Exchange: @retrier def fetch_ticker(self, pair: str) -> dict: try: - if pair not in self._api.markets or not self._api.markets[pair].get('active'): + if (pair not in self._api.markets or + self._api.markets[pair].get('active', False) is False): raise ExchangeError(f"Pair {pair} not available") data = self._api.fetch_ticker(pair) return data @@ -732,13 +733,17 @@ class Exchange: logger.info("Downloaded data for %s with length %s.", pair, len(data)) return data - def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes) -> List[Tuple[str, List]]: + def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, + since_ms: Optional[int] = None, cache: bool = True + ) -> Dict[Tuple[str, str], DataFrame]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). Only used in the dataprovider.refresh() method. :param pair_list: List of 2 element tuples containing pair, interval to refresh - :return: TODO: return value is only used in the tests, get rid of it + :param since_ms: time since when to download, in milliseconds + :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists + :return: Dict of [{(pair, timeframe): Dataframe}] """ logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) @@ -748,7 +753,8 @@ class Exchange: for pair, timeframe in set(pair_list): if (not ((pair, timeframe) in self._klines) or self._now_is_time_to_refresh(pair, timeframe)): - input_coroutines.append(self._async_get_candle_history(pair, timeframe)) + input_coroutines.append(self._async_get_candle_history(pair, timeframe, + since_ms=since_ms)) else: logger.debug( "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", @@ -758,6 +764,7 @@ class Exchange: results = asyncio.get_event_loop().run_until_complete( asyncio.gather(*input_coroutines, return_exceptions=True)) + results_df = {} # handle caching for res in results: if isinstance(res, Exception): @@ -769,11 +776,13 @@ class Exchange: if ticks: self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache - self._klines[(pair, timeframe)] = ohlcv_to_dataframe( - ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) - - return results + ohlcv_df = ohlcv_to_dataframe( + ticks, timeframe, pair=pair, fill_missing=True, + drop_incomplete=self._ohlcv_partial_candle) + results_df[(pair, timeframe)] = ohlcv_df + if cache: + self._klines[(pair, timeframe)] = ohlcv_df + return results_df def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool: # Timeframe in seconds @@ -798,7 +807,8 @@ class Exchange: ) data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, - since=since_ms) + since=since_ms, + limit=self._ohlcv_candle_limit) # Some exchanges sort OHLCV in ASC order and others in DESC. # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 4e4713052..6dbb751e5 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -18,6 +18,7 @@ class Kraken(Exchange): _params: Dict = {"trading_agreement": "agree"} _ft_has: Dict = { "stoploss_on_exchange": True, + "ohlcv_candle_limit": 720, "trades_pagination": "id", "trades_pagination_arg": "since", } diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7416d8236..d60b111f2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -19,10 +19,12 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) -from freqtrade.exchange import timeframe_to_minutes +from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.misc import safe_value_fallback, safe_value_fallback2 -from freqtrade.pairlist.pairlistmanager import PairListManager +from freqtrade.mixins import LoggingMixin from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db +from freqtrade.plugins.pairlistmanager import PairListManager +from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State @@ -34,7 +36,7 @@ from freqtrade.wallets import Wallets logger = logging.getLogger(__name__) -class FreqtradeBot: +class FreqtradeBot(LoggingMixin): """ Freqtrade is the main class of the bot. This is from here the bot start its logic. @@ -78,6 +80,8 @@ class FreqtradeBot: self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) + self.protections = ProtectionManager(self.config) + # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass @@ -101,6 +105,7 @@ class FreqtradeBot: self.rpc: RPCManager = RPCManager(self) # Protect sell-logic from forcesell and viceversa self._sell_lock = Lock() + LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) def notify_status(self, msg: str) -> None: """ @@ -132,7 +137,7 @@ class FreqtradeBot: Called on startup and after reloading the bot - triggers notifications and performs startup tasks """ - self.rpc.startup_messages(self.config, self.pairlists) + self.rpc.startup_messages(self.config, self.pairlists, self.protections) if not self.edge: # Adjust stoploss if it was changed Trade.stoploss_reinitialization(self.strategy.stoploss) @@ -358,6 +363,15 @@ class FreqtradeBot: logger.info("No currency pair in active pair whitelist, " "but checking to sell open trades.") return trades_created + if PairLocks.is_global_lock(): + lock = PairLocks.get_pair_longest_lock('*') + if lock: + self.log_once(f"Global pairlock active until " + f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}. " + "Not creating new trades.", logger.info) + else: + self.log_once("Global pairlock active. Not creating new trades.", logger.info) + return trades_created # Create entity and execute trade for each pair from whitelist for pair in whitelist: try: @@ -366,8 +380,7 @@ class FreqtradeBot: logger.warning('Unable to create trade for %s: %s', pair, exception) if not trades_created: - logger.debug("Found no buy signals for whitelisted currencies. " - "Trying again...") + logger.debug("Found no buy signals for whitelisted currencies. Trying again...") return trades_created @@ -519,8 +532,7 @@ class FreqtradeBot: # reserve some percent defined in config (5% default) + stoploss amount_reserve_percent = 1.0 - self.config.get('amount_reserve_percent', constants.DEFAULT_AMOUNT_RESERVE_PERCENT) - if self.strategy.stoploss is not None: - amount_reserve_percent += self.strategy.stoploss + amount_reserve_percent += self.strategy.stoploss # it should not be more than 50% amount_reserve_percent = max(amount_reserve_percent, 0.5) @@ -541,9 +553,15 @@ class FreqtradeBot: logger.debug(f"create_trade for pair {pair}") analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(pair, self.strategy.timeframe) - if self.strategy.is_pair_locked( - pair, analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None): - logger.info(f"Pair {pair} is currently locked.") + nowtime = analyzed_df.iloc[-1]['date'] if len(analyzed_df) > 0 else None + if self.strategy.is_pair_locked(pair, nowtime): + lock = PairLocks.get_pair_longest_lock(pair, nowtime) + if lock: + self.log_once(f"Pair {pair} is still locked until " + f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}.", + logger.info) + else: + self.log_once(f"Pair {pair} is still locked.", logger.info) return False # get_free_open_trades is checked before create_trade is called @@ -616,6 +634,9 @@ class FreqtradeBot: # Calculate price buy_limit_requested = self.get_buy_rate(pair, True) + if not buy_limit_requested: + raise PricingError('Could not determine buy price.') + min_stake_amount = self._get_min_pair_stake_amount(pair, buy_limit_requested) if min_stake_amount is not None and min_stake_amount > stake_amount: logger.warning( @@ -1393,7 +1414,7 @@ class FreqtradeBot: abs_tol=constants.MATH_CLOSE_PREC): order['amount'] = new_amount order.pop('filled', None) - trade.recalc_open_trade_price() + trade.recalc_open_trade_value() except DependencyException as exception: logger.warning("Could not update trade amount: %s", exception) @@ -1405,6 +1426,8 @@ class FreqtradeBot: # Updating wallets when order is closed if not trade.is_open: + self.protections.stop_per_pair(trade.pair) + self.protections.global_stop() self.wallets.update() return False @@ -1446,13 +1469,16 @@ class FreqtradeBot: fee_cost, fee_currency, fee_rate = self.exchange.extract_cost_curr_rate(order) logger.info(f"Fee for Trade {trade} [{order.get('side')}]: " f"{fee_cost:.8g} {fee_currency} - rate: {fee_rate}") - - trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) - if trade_base_currency == fee_currency: - # Apply fee to amount - return self.apply_fee_conditional(trade, trade_base_currency, - amount=order_amount, fee_abs=fee_cost) - return order_amount + if fee_rate is None or fee_rate < 0.02: + # Reject all fees that report as > 2%. + # These are most likely caused by a parsing bug in ccxt + # due to multiple trades (https://github.com/ccxt/ccxt/issues/8025) + trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) + if trade_base_currency == fee_currency: + # Apply fee to amount + return self.apply_fee_conditional(trade, trade_base_currency, + amount=order_amount, fee_abs=fee_cost) + return order_amount return self.fee_detection_from_trades(trade, order, order_amount) def fee_detection_from_trades(self, trade: Trade, order: Dict, order_amount: float) -> float: diff --git a/freqtrade/main.py b/freqtrade/main.py index 5f8d5d19d..84d4b24f8 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -9,8 +9,8 @@ from typing import Any, List # check min. python version -if sys.version_info < (3, 6): - sys.exit("Freqtrade requires Python version >= 3.6") +if sys.version_info < (3, 7): + sys.exit("Freqtrade requires Python version >= 3.7") from freqtrade.commands import Arguments from freqtrade.exceptions import FreqtradeException, OperationalException diff --git a/freqtrade/mixins/__init__.py b/freqtrade/mixins/__init__.py new file mode 100644 index 000000000..f4a640fa3 --- /dev/null +++ b/freqtrade/mixins/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa: F401 +from freqtrade.mixins.logging_mixin import LoggingMixin diff --git a/freqtrade/mixins/logging_mixin.py b/freqtrade/mixins/logging_mixin.py new file mode 100644 index 000000000..06935d5f6 --- /dev/null +++ b/freqtrade/mixins/logging_mixin.py @@ -0,0 +1,38 @@ +from typing import Callable + +from cachetools import TTLCache, cached + + +class LoggingMixin(): + """ + Logging Mixin + Shows similar messages only once every `refresh_period`. + """ + # Disable output completely + show_output = True + + def __init__(self, logger, refresh_period: int = 3600): + """ + :param refresh_period: in seconds - Show identical messages in this intervals + """ + self.logger = logger + self.refresh_period = refresh_period + self._log_cache: TTLCache = TTLCache(maxsize=1024, ttl=self.refresh_period) + + def log_once(self, message: str, logmethod: Callable) -> None: + """ + Logs message - not more often than "refresh_period" to avoid log spamming + Logs the log-message as debug as well to simplify debugging. + :param message: String containing the message to be sent to the function. + :param logmethod: Function that'll be called. Most likely `logger.info`. + :return: None. + """ + @cached(cache=self._log_cache) + def _log_once(message: str): + logmethod(message) + + # Log as debug first + self.logger.debug(message) + # Call hidden function. + if self.show_output: + _log_once(message) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 883f7338c..a689786ec 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -18,10 +18,12 @@ from freqtrade.data.converter import trim_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds +from freqtrade.mixins import LoggingMixin from freqtrade.optimize.optimize_reports import (generate_backtest_stats, show_backtest_results, store_backtest_stats) -from freqtrade.pairlist.pairlistmanager import PairListManager -from freqtrade.persistence import Trade +from freqtrade.persistence import PairLocks, Trade +from freqtrade.plugins.pairlistmanager import PairListManager +from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType @@ -67,6 +69,8 @@ class Backtesting: """ def __init__(self, config: Dict[str, Any]) -> None: + + LoggingMixin.show_output = False self.config = config # Reset keys for backtesting @@ -98,6 +102,8 @@ class Backtesting: self.pairlists = PairListManager(self.exchange, self.config) if 'VolumePairList' in self.pairlists.name_list: raise OperationalException("VolumePairList not allowed for backtesting.") + if 'PerformanceFilter' in self.pairlists.name_list: + raise OperationalException("PerformanceFilter not allowed for backtesting.") if len(self.strategylist) > 1 and 'PrecisionFilter' in self.pairlists.name_list: raise OperationalException( @@ -115,11 +121,24 @@ class Backtesting: else: self.fee = self.exchange.get_fee(symbol=self.pairlists.whitelist[0]) + Trade.use_db = False + Trade.reset_trades() + PairLocks.timeframe = self.config['timeframe'] + PairLocks.use_db = False + PairLocks.reset_locks() + if self.config.get('enable_protections', False): + self.protections = ProtectionManager(self.config) + # Get maximum required startup period self.required_startup = max([strat.startup_candle_count for strat in self.strategylist]) # Load one (first) strategy self._set_strategy(self.strategylist[0]) + def __del__(self): + LoggingMixin.show_output = True + PairLocks.use_db = True + Trade.use_db = True + def _set_strategy(self, strategy): """ Load strategy into backtesting @@ -156,6 +175,17 @@ class Backtesting: return data, timerange + def prepare_backtest(self, enable_protections): + """ + Backtesting setup method - called once for every call to "backtest()". + """ + PairLocks.use_db = False + Trade.use_db = False + if enable_protections: + # Reset persisted data - used for protections only + PairLocks.reset_locks() + Trade.reset_trades() + def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]: """ Helper function to convert a processed dataframes into lists for performance reasons. @@ -235,6 +265,10 @@ class Backtesting: trade_dur = int((sell_row[DATE_IDX] - trade.open_date).total_seconds() // 60) closerate = self._get_close_rate(sell_row, trade, sell, trade_dur) + trade.close_date = sell_row[DATE_IDX] + trade.sell_reason = sell.sell_type + trade.close(closerate, show_msg=False) + return BacktestResult(pair=trade.pair, profit_percent=trade.calc_profit_ratio(rate=closerate), profit_abs=trade.calc_profit(rate=closerate), @@ -261,6 +295,7 @@ class Backtesting: if len(open_trades[pair]) > 0: for trade in open_trades[pair]: sell_row = data[pair][-1] + trade_entry = BacktestResult(pair=trade.pair, profit_percent=trade.calc_profit_ratio( rate=sell_row[OPEN_IDX]), @@ -283,7 +318,8 @@ class Backtesting: def backtest(self, processed: Dict, stake_amount: float, start_date: datetime, end_date: datetime, - max_open_trades: int = 0, position_stacking: bool = False) -> DataFrame: + max_open_trades: int = 0, position_stacking: bool = False, + enable_protections: bool = False) -> DataFrame: """ Implement backtesting functionality @@ -297,6 +333,7 @@ class Backtesting: :param end_date: backtesting timerange end datetime :param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited :param position_stacking: do we allow position stacking? + :param enable_protections: Should protections be enabled? :return: DataFrame with trades (results of backtesting) """ logger.debug(f"Run backtest, stake_amount: {stake_amount}, " @@ -304,6 +341,7 @@ class Backtesting: f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}" ) trades = [] + self.prepare_backtest(enable_protections) # Use dict of lists with data for performance # (looping lists is a lot faster than pandas DataFrames) @@ -342,7 +380,8 @@ class Backtesting: if ((position_stacking or len(open_trades[pair]) == 0) and (max_open_trades <= 0 or open_trade_count_start < max_open_trades) and tmp != end_date - and row[BUY_IDX] == 1 and row[SELL_IDX] != 1): + and row[BUY_IDX] == 1 and row[SELL_IDX] != 1 + and not PairLocks.is_pair_locked(pair, row[DATE_IDX])): # Enter trade trade = Trade( pair=pair, @@ -361,6 +400,7 @@ class Backtesting: open_trade_count += 1 # logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.") open_trades[pair].append(trade) + Trade.trades.append(trade) for trade in open_trades[pair]: # since indexes has been incremented before, we need to go one step back to @@ -372,6 +412,9 @@ class Backtesting: open_trade_count -= 1 open_trades[pair].remove(trade) trades.append(trade_entry) + if enable_protections: + self.protections.stop_per_pair(pair, row[DATE_IDX]) + self.protections.global_stop(tmp) # Move time one configured time_interval ahead. tmp += timedelta(minutes=self.timeframe_min) @@ -427,10 +470,12 @@ class Backtesting: end_date=max_date.datetime, max_open_trades=max_open_trades, position_stacking=position_stacking, + enable_protections=self.config.get('enable_protections', False), ) all_results[self.strategy.get_strategy_name()] = { 'results': results, 'config': self.strategy.config, + 'locks': PairLocks.locks, } stats = generate_backtest_stats(data, all_results, min_date=min_date, max_date=max_date) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 7870ba1cf..2a2f5b472 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -542,6 +542,8 @@ class Hyperopt: end_date=max_date.datetime, max_open_trades=self.max_open_trades, position_stacking=self.position_stacking, + enable_protections=self.config.get('enable_protections', False), + ) return self._get_results_dict(backtesting_results, min_date, max_date, params_dict, params_details) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index fc04cbd93..d029ecd13 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -58,16 +58,19 @@ def _generate_result_line(result: DataFrame, max_open_trades: int, first_column: """ Generate one result dict, with "first_column" as key. """ + profit_sum = result['profit_percent'].sum() + profit_total = profit_sum / max_open_trades + return { 'key': first_column, 'trades': len(result), 'profit_mean': result['profit_percent'].mean() if len(result) > 0 else 0.0, 'profit_mean_pct': result['profit_percent'].mean() * 100.0 if len(result) > 0 else 0.0, - 'profit_sum': result['profit_percent'].sum(), - 'profit_sum_pct': result['profit_percent'].sum() * 100.0, + 'profit_sum': profit_sum, + 'profit_sum_pct': round(profit_sum * 100.0, 2), 'profit_total_abs': result['profit_abs'].sum(), - 'profit_total': result['profit_percent'].sum() / max_open_trades, - 'profit_total_pct': result['profit_percent'].sum() * 100.0 / max_open_trades, + 'profit_total': profit_total, + 'profit_total_pct': round(profit_total * 100.0, 2), 'duration_avg': str(timedelta( minutes=round(result['trade_duration'].mean())) ) if not result.empty else '0:00', @@ -122,8 +125,8 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List result = results.loc[results['sell_reason'] == reason] profit_mean = result['profit_percent'].mean() - profit_sum = result["profit_percent"].sum() - profit_percent_tot = result['profit_percent'].sum() / max_open_trades + profit_sum = result['profit_percent'].sum() + profit_total = profit_sum / max_open_trades tabular_data.append( { @@ -137,8 +140,8 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List 'profit_sum': profit_sum, 'profit_sum_pct': round(profit_sum * 100, 2), 'profit_total_abs': result['profit_abs'].sum(), - 'profit_total': profit_percent_tot, - 'profit_total_pct': round(profit_percent_tot * 100, 2), + 'profit_total': profit_total, + 'profit_total_pct': round(profit_total * 100, 2), } ) return tabular_data @@ -253,13 +256,19 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame], results=results.loc[results['open_at_end']], skip_nan=True) daily_stats = generate_daily_stats(results) - + best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'], + key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None + worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'], + key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None results['open_timestamp'] = results['open_date'].astype(int64) // 1e6 results['close_timestamp'] = results['close_date'].astype(int64) // 1e6 backtest_days = (max_date - min_date).days strat_stats = { 'trades': results.to_dict(orient='records'), + 'locks': [lock.to_json() for lock in content['locks']], + 'best_pair': best_pair, + 'worst_pair': worst_pair, 'results_per_pair': pair_results, 'sell_reason_summary': sell_reason_stats, 'left_open_trades': left_open_results, @@ -392,17 +401,25 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str: def text_table_add_metrics(strat_results: Dict) -> str: if len(strat_results['trades']) > 0: - min_trade = min(strat_results['trades'], key=lambda x: x['open_date']) + best_trade = max(strat_results['trades'], key=lambda x: x['profit_percent']) + worst_trade = min(strat_results['trades'], key=lambda x: x['profit_percent']) metrics = [ ('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)), ('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)), ('Max open trades', strat_results['max_open_trades']), ('', ''), # Empty line to improve readability ('Total trades', strat_results['total_trades']), - ('First trade', min_trade['open_date'].strftime(DATETIME_PRINT_FORMAT)), - ('First trade Pair', min_trade['pair']), ('Total Profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"), ('Trades per day', strat_results['trades_per_day']), + ('', ''), # Empty line to improve readability + ('Best Pair', f"{strat_results['best_pair']['key']} " + f"{round(strat_results['best_pair']['profit_sum_pct'], 2)}%"), + ('Worst Pair', f"{strat_results['worst_pair']['key']} " + f"{round(strat_results['worst_pair']['profit_sum_pct'], 2)}%"), + ('Best trade', f"{best_trade['pair']} {round(best_trade['profit_percent'] * 100, 2)}%"), + ('Worst trade', f"{worst_trade['pair']} " + f"{round(worst_trade['profit_percent'] * 100, 2)}%"), + ('Best day', f"{round(strat_results['backtest_best_day'] * 100, 2)}%"), ('Worst day', f"{round(strat_results['backtest_worst_day'] * 100, 2)}%"), ('Days win/draw/lose', f"{strat_results['winning_days']} / " diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index 84f3ed7e6..ed976c2a9 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -53,11 +53,11 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col else: timeframe = get_column_def(cols, 'timeframe', 'null') - open_trade_price = get_column_def(cols, 'open_trade_price', + open_trade_value = get_column_def(cols, 'open_trade_value', f'amount * open_rate * (1 + {fee_open})') close_profit_abs = get_column_def( cols, 'close_profit_abs', - f"(amount * close_rate * (1 - {fee_close})) - {open_trade_price}") + f"(amount * close_rate * (1 - {fee_close})) - {open_trade_value}") sell_order_status = get_column_def(cols, 'sell_order_status', 'null') amount_requested = get_column_def(cols, 'amount_requested', 'amount') @@ -79,7 +79,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col stop_loss, stop_loss_pct, initial_stop_loss, initial_stop_loss_pct, stoploss_order_id, stoploss_last_update, max_rate, min_rate, sell_reason, sell_order_status, strategy, - timeframe, open_trade_price, close_profit_abs + timeframe, open_trade_value, close_profit_abs ) select id, lower(exchange), case @@ -102,7 +102,7 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col {max_rate} max_rate, {min_rate} min_rate, {sell_reason} sell_reason, {sell_order_status} sell_order_status, {strategy} strategy, {timeframe} timeframe, - {open_trade_price} open_trade_price, {close_profit_abs} close_profit_abs + {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs from {table_back_name} """) @@ -134,7 +134,7 @@ def check_migrate(engine, decl_base, previous_tables) -> None: table_back_name = get_backup_name(tabs, 'trades_bak') # Check for latest column - if not has_column(cols, 'amount_requested'): + if not has_column(cols, 'open_trade_value'): logger.info(f'Running database migration for trades - backup: {table_back_name}') migrate_trades_table(decl_base, inspector, engine, table_back_name, cols) # Reread columns - the above recreated the table! diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 6027908da..7fa894e9c 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -202,6 +202,10 @@ class Trade(_DECL_BASE): """ __tablename__ = 'trades' + use_db: bool = True + # Trades container for backtesting + trades: List['Trade'] = [] + id = Column(Integer, primary_key=True) orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan") @@ -217,8 +221,8 @@ class Trade(_DECL_BASE): fee_close_currency = Column(String, nullable=True) open_rate = Column(Float) open_rate_requested = Column(Float) - # open_trade_price - calculated via _calc_open_trade_price - open_trade_price = Column(Float) + # open_trade_value - calculated via _calc_open_trade_value + open_trade_value = Column(Float) close_rate = Column(Float) close_rate_requested = Column(Float) close_profit = Column(Float) @@ -252,7 +256,7 @@ class Trade(_DECL_BASE): def __init__(self, **kwargs): super().__init__(**kwargs) - self.recalc_open_trade_price() + self.recalc_open_trade_value() def __repr__(self): open_since = self.open_date.strftime(DATETIME_PRINT_FORMAT) if self.is_open else 'closed' @@ -284,7 +288,7 @@ class Trade(_DECL_BASE): 'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000), 'open_rate': self.open_rate, 'open_rate_requested': self.open_rate_requested, - 'open_trade_price': round(self.open_trade_price, 8), + 'open_trade_value': round(self.open_trade_value, 8), 'close_date_hum': (arrow.get(self.close_date).humanize() if self.close_date else None), @@ -323,6 +327,14 @@ class Trade(_DECL_BASE): 'open_order_id': self.open_order_id, } + @staticmethod + def reset_trades() -> None: + """ + Resets all trades. Only active for backtesting mode. + """ + if not Trade.use_db: + Trade.trades = [] + def adjust_min_max_rates(self, current_price: float) -> None: """ Adjust the max_rate and min_rate. @@ -389,7 +401,7 @@ class Trade(_DECL_BASE): # Update open rate and actual amount self.open_rate = Decimal(safe_value_fallback(order, 'average', 'price')) self.amount = Decimal(safe_value_fallback(order, 'filled', 'amount')) - self.recalc_open_trade_price() + self.recalc_open_trade_value() if self.is_open: logger.info(f'{order_type.upper()}_BUY has been fulfilled for {self}.') self.open_order_id = None @@ -407,7 +419,7 @@ class Trade(_DECL_BASE): raise ValueError(f'Unknown order type: {order_type}') cleanup_db() - def close(self, rate: float) -> None: + def close(self, rate: float, *, show_msg: bool = True) -> None: """ Sets close_rate to the given rate, calculates total profit and marks trade as closed @@ -419,10 +431,11 @@ class Trade(_DECL_BASE): self.is_open = False self.sell_order_status = 'closed' self.open_order_id = None - logger.info( - 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', - self - ) + if show_msg: + logger.info( + 'Marking %s as closed as the trade is fulfilled and found no open orders for it.', + self + ) def update_fee(self, fee_cost: float, fee_currency: Optional[str], fee_rate: Optional[float], side: str) -> None: @@ -464,7 +477,7 @@ class Trade(_DECL_BASE): Trade.session.delete(self) Trade.session.flush() - def _calc_open_trade_price(self) -> float: + def _calc_open_trade_value(self) -> float: """ Calculate the open_rate including open_fee. :return: Price in of the open trade incl. Fees @@ -473,14 +486,14 @@ class Trade(_DECL_BASE): fees = buy_trade * Decimal(self.fee_open) return float(buy_trade + fees) - def recalc_open_trade_price(self) -> None: + def recalc_open_trade_value(self) -> None: """ - Recalculate open_trade_price. + Recalculate open_trade_value. Must be called whenever open_rate or fee_open is changed. """ - self.open_trade_price = self._calc_open_trade_price() + self.open_trade_value = self._calc_open_trade_value() - def calc_close_trade_price(self, rate: Optional[float] = None, + def calc_close_trade_value(self, rate: Optional[float] = None, fee: Optional[float] = None) -> float: """ Calculate the close_rate including fee @@ -507,11 +520,11 @@ class Trade(_DECL_BASE): If rate is not set self.close_rate will be used :return: profit in stake currency as float """ - close_trade_price = self.calc_close_trade_price( + close_trade_value = self.calc_close_trade_value( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - profit = close_trade_price - self.open_trade_price + profit = close_trade_value - self.open_trade_value return float(f"{profit:.8f}") def calc_profit_ratio(self, rate: Optional[float] = None, @@ -523,11 +536,11 @@ class Trade(_DECL_BASE): :param fee: fee to use on the close rate (optional). :return: profit ratio as float """ - close_trade_price = self.calc_close_trade_price( + close_trade_value = self.calc_close_trade_value( rate=(rate or self.close_rate), fee=(fee or self.fee_close) ) - profit_ratio = (close_trade_price / self.open_trade_price) - 1 + profit_ratio = (close_trade_value / self.open_trade_value) - 1 return float(f"{profit_ratio:.8f}") def select_order(self, order_side: str, is_open: Optional[bool]) -> Optional[Order]: @@ -562,6 +575,43 @@ class Trade(_DECL_BASE): else: return Trade.query + @staticmethod + def get_trades_proxy(*, pair: str = None, is_open: bool = None, + open_date: datetime = None, close_date: datetime = None, + ) -> List['Trade']: + """ + Helper function to query Trades. + Returns a List of trades, filtered on the parameters given. + In live mode, converts the filter to a database query and returns all rows + In Backtest mode, uses filters on Trade.trades to get the result. + + :return: unsorted List[Trade] + """ + if Trade.use_db: + trade_filter = [] + if pair: + trade_filter.append(Trade.pair == pair) + if open_date: + trade_filter.append(Trade.open_date > open_date) + if close_date: + trade_filter.append(Trade.close_date > close_date) + if is_open is not None: + trade_filter.append(Trade.is_open.is_(is_open)) + return Trade.get_trades(trade_filter).all() + else: + # Offline mode - without database + sel_trades = [trade for trade in Trade.trades] + if pair: + sel_trades = [trade for trade in sel_trades if trade.pair == pair] + if open_date: + sel_trades = [trade for trade in sel_trades if trade.open_date > open_date] + if close_date: + sel_trades = [trade for trade in sel_trades if trade.close_date + and trade.close_date > close_date] + if is_open is not None: + sel_trades = [trade for trade in sel_trades if trade.is_open == is_open] + return sel_trades + @staticmethod def get_open_trades() -> List[Any]: """ @@ -688,7 +738,7 @@ class PairLock(_DECL_BASE): @staticmethod def query_pair_locks(pair: Optional[str], now: datetime) -> Query: """ - Get all locks for this pair + Get all currently active locks for this pair :param pair: Pair to check for. Returns all current locks if pair is empty :param now: Datetime object (generated via datetime.now(timezone.utc)). """ diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index 44fc228f6..8644146d8 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -22,10 +22,27 @@ class PairLocks(): timeframe: str = '' @staticmethod - def lock_pair(pair: str, until: datetime, reason: str = None) -> None: + def reset_locks() -> None: + """ + Resets all locks. Only active for backtesting mode. + """ + if not PairLocks.use_db: + PairLocks.locks = [] + + @staticmethod + def lock_pair(pair: str, until: datetime, reason: str = None, *, now: datetime = None) -> None: + """ + Create PairLock from now to "until". + Uses database by default, unless PairLocks.use_db is set to False, + in which case a list is maintained. + :param pair: pair to lock. use '*' to lock all pairs + :param until: End time of the lock. Will be rounded up to the next candle. + :param reason: Reason string that will be shown as reason for the lock + :param now: Current timestamp. Used to determine lock start time. + """ lock = PairLock( pair=pair, - lock_time=datetime.now(timezone.utc), + lock_time=now or datetime.now(timezone.utc), lock_end_time=timeframe_to_next_date(PairLocks.timeframe, until), reason=reason, active=True @@ -57,6 +74,15 @@ class PairLocks(): )] return locks + @staticmethod + def get_pair_longest_lock(pair: str, now: Optional[datetime] = None) -> Optional[PairLock]: + """ + Get the lock that expires the latest for the pair given. + """ + locks = PairLocks.get_pair_locks(pair, now) + locks = sorted(locks, key=lambda l: l.lock_end_time, reverse=True) + return locks[0] if locks else None + @staticmethod def unlock_pair(pair: str, now: Optional[datetime] = None) -> None: """ diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index f7d300593..497218deb 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -263,6 +263,65 @@ def create_plotconfig(indicators1: List[str], indicators2: List[str], return plot_config +def plot_area(fig, row: int, data: pd.DataFrame, indicator_a: str, + indicator_b: str, label: str = "", + fill_color: str = "rgba(0,176,246,0.2)") -> make_subplots: + """ Creates a plot for the area between two traces and adds it to fig. + :param fig: Plot figure to append to + :param row: row number for this plot + :param data: candlestick DataFrame + :param indicator_a: indicator name as populated in stragetie + :param indicator_b: indicator name as populated in stragetie + :param label: label for the filled area + :param fill_color: color to be used for the filled area + :return: fig with added filled_traces plot + """ + if indicator_a in data and indicator_b in data: + # make lines invisible to get the area plotted, only. + line = {'color': 'rgba(255,255,255,0)'} + # TODO: Figure out why scattergl causes problems plotly/plotly.js#2284 + trace_a = go.Scatter(x=data.date, y=data[indicator_a], + showlegend=False, + line=line) + trace_b = go.Scatter(x=data.date, y=data[indicator_b], name=label, + fill="tonexty", fillcolor=fill_color, + line=line) + fig.add_trace(trace_a, row, 1) + fig.add_trace(trace_b, row, 1) + return fig + + +def add_areas(fig, row: int, data: pd.DataFrame, indicators) -> make_subplots: + """ Adds all area plots (specified in plot_config) to fig. + :param fig: Plot figure to append to + :param row: row number for this plot + :param data: candlestick DataFrame + :param indicators: dict with indicators. ie.: plot_config['main_plot'] or + plot_config['subplots'][subplot_label] + :return: fig with added filled_traces plot + """ + for indicator, ind_conf in indicators.items(): + if 'fill_to' in ind_conf: + indicator_b = ind_conf['fill_to'] + if indicator in data and indicator_b in data: + label = ind_conf.get('fill_label', + f'{indicator}<>{indicator_b}') + fill_color = ind_conf.get('fill_color', 'rgba(0,176,246,0.2)') + fig = plot_area(fig, row, data, indicator, indicator_b, + label=label, fill_color=fill_color) + elif indicator not in data: + logger.info( + 'Indicator "%s" ignored. Reason: This indicator is not ' + 'found in your strategy.', indicator + ) + elif indicator_b not in data: + logger.info( + 'fill_to: "%s" ignored. Reason: This indicator is not ' + 'in your strategy.', indicator_b + ) + return fig + + def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFrame = None, *, indicators1: List[str] = [], indicators2: List[str] = [], @@ -280,7 +339,6 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra :return: Plotly figure """ plot_config = create_plotconfig(indicators1, indicators2, plot_config) - rows = 2 + len(plot_config['subplots']) row_widths = [1 for _ in plot_config['subplots']] # Define the graph @@ -346,36 +404,20 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra fig.add_trace(sells, 1, 1) else: logger.warning("No sell-signals found.") - - # TODO: Figure out why scattergl causes problems plotly/plotly.js#2284 - if 'bb_lowerband' in data and 'bb_upperband' in data: - bb_lower = go.Scatter( - x=data.date, - y=data.bb_lowerband, - showlegend=False, - line={'color': 'rgba(255,255,255,0)'}, - ) - bb_upper = go.Scatter( - x=data.date, - y=data.bb_upperband, - name='Bollinger Band', - fill="tonexty", - fillcolor="rgba(0,176,246,0.2)", - line={'color': 'rgba(255,255,255,0)'}, - ) - fig.add_trace(bb_lower, 1, 1) - fig.add_trace(bb_upper, 1, 1) - if ('bb_upperband' in plot_config['main_plot'] - and 'bb_lowerband' in plot_config['main_plot']): - del plot_config['main_plot']['bb_upperband'] - del plot_config['main_plot']['bb_lowerband'] - - # Add indicators to main plot + # Add Bollinger Bands + fig = plot_area(fig, 1, data, 'bb_lowerband', 'bb_upperband', + label="Bollinger Band") + # prevent bb_lower and bb_upper from plotting + try: + del plot_config['main_plot']['bb_lowerband'] + del plot_config['main_plot']['bb_upperband'] + except KeyError: + pass + # main plot goes to row 1 fig = add_indicators(fig=fig, row=1, indicators=plot_config['main_plot'], data=data) - + fig = add_areas(fig, 1, data, plot_config['main_plot']) fig = plot_trades(fig, trades) - - # Volume goes to row 2 + # sub plot: Volume goes to row 2 volume = go.Bar( x=data['date'], y=data['volume'], @@ -384,13 +426,14 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra marker_line_color='DarkSlateGrey' ) fig.add_trace(volume, 2, 1) - - # Add indicators to separate row - for i, name in enumerate(plot_config['subplots']): - fig = add_indicators(fig=fig, row=3 + i, - indicators=plot_config['subplots'][name], + # add each sub plot to a separate row + for i, label in enumerate(plot_config['subplots']): + sub_config = plot_config['subplots'][label] + row = 3 + i + fig = add_indicators(fig=fig, row=row, indicators=sub_config, data=data) - + # fill area between indicators ( 'fill_to': 'other_indicator') + fig = add_areas(fig, row, data, sub_config) return fig diff --git a/freqtrade/pairlist/__init__.py b/freqtrade/plugins/__init__.py similarity index 100% rename from freqtrade/pairlist/__init__.py rename to freqtrade/plugins/__init__.py diff --git a/freqtrade/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py similarity index 58% rename from freqtrade/pairlist/AgeFilter.py rename to freqtrade/plugins/pairlist/AgeFilter.py index e2a13c20a..8c3a5d22f 100644 --- a/freqtrade/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -2,13 +2,15 @@ Minimum age (days listed) pair list filter """ import logging -from typing import Any, Dict +from copy import deepcopy +from typing import Any, Dict, List, Optional import arrow +from pandas import DataFrame from freqtrade.exceptions import OperationalException from freqtrade.misc import plural -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -40,7 +42,7 @@ class AgeFilter(IPairList): If no Pairlist requires tickers, an empty Dict is passed as tickers argument to filter_pairlist """ - return True + return False def short_desc(self) -> str: """ @@ -49,36 +51,49 @@ class AgeFilter(IPairList): return (f"{self.name} - Filtering pairs with age less than " f"{self._min_days_listed} {plural(self._min_days_listed, 'day')}.") - def _validate_pair(self, ticker: Dict) -> bool: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ - Validate age for the ticker - :param ticker: ticker dict as returned from ccxt.load_markets() - :return: True if the pair can stay, False if it should be removed + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist """ - - # Check symbol in cache - if ticker['symbol'] in self._symbolsChecked: - return True + needed_pairs = [(p, '1d') for p in pairlist if p not in self._symbolsChecked] + if not needed_pairs: + return pairlist since_ms = int(arrow.utcnow() .floor('day') - .shift(days=-self._min_days_listed) + .shift(days=-self._min_days_listed - 1) .float_timestamp) * 1000 + candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) + if self._enabled: + for p in deepcopy(pairlist): + daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None + if not self._validate_pair_loc(p, daily_candles): + pairlist.remove(p) + logger.info(f"Validated {len(pairlist)} pairs.") + return pairlist - daily_candles = self._exchange.get_historic_ohlcv(pair=ticker['symbol'], - timeframe='1d', - since_ms=since_ms) + def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: + """ + Validate age for the ticker + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, false if it should be removed + """ + # Check symbol in cache + if pair in self._symbolsChecked: + return True if daily_candles is not None: if len(daily_candles) > self._min_days_listed: # We have fetched at least the minimum required number of daily candles # Add to cache, store the time we last checked this symbol - self._symbolsChecked[ticker['symbol']] = int(arrow.utcnow().float_timestamp) * 1000 + self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000 return True else: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because age {len(daily_candles)} is less than " - f"{self._min_days_listed} " - f"{plural(self._min_days_listed, 'day')}") + self.log_once(f"Removed {pair} from whitelist, because age " + f"{len(daily_candles)} is less than {self._min_days_listed} " + f"{plural(self._min_days_listed, 'day')}", logger.info) return False return False diff --git a/freqtrade/pairlist/IPairList.py b/freqtrade/plugins/pairlist/IPairList.py similarity index 87% rename from freqtrade/pairlist/IPairList.py rename to freqtrade/plugins/pairlist/IPairList.py index c869e499b..865aa90d6 100644 --- a/freqtrade/pairlist/IPairList.py +++ b/freqtrade/plugins/pairlist/IPairList.py @@ -6,16 +6,15 @@ from abc import ABC, abstractmethod, abstractproperty from copy import deepcopy from typing import Any, Dict, List -from cachetools import TTLCache, cached - from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active +from freqtrade.mixins import LoggingMixin logger = logging.getLogger(__name__) -class IPairList(ABC): +class IPairList(LoggingMixin, ABC): def __init__(self, exchange, pairlistmanager, config: Dict[str, Any], pairlistconfig: Dict[str, Any], @@ -36,7 +35,7 @@ class IPairList(ABC): self._pairlist_pos = pairlist_pos self.refresh_period = self._pairlistconfig.get('refresh_period', 1800) self._last_refresh = 0 - self._log_cache: TTLCache = TTLCache(maxsize=1024, ttl=self.refresh_period) + LoggingMixin.__init__(self, logger, self.refresh_period) @property def name(self) -> str: @@ -46,24 +45,6 @@ class IPairList(ABC): """ return self.__class__.__name__ - def log_on_refresh(self, logmethod, message: str) -> None: - """ - Logs message - not more often than "refresh_period" to avoid log spamming - Logs the log-message as debug as well to simplify debugging. - :param logmethod: Function that'll be called. Most likely `logger.info`. - :param message: String containing the message to be sent to the function. - :return: None. - """ - - @cached(cache=self._log_cache) - def _log_on_refresh(message: str): - logmethod(message) - - # Log as debug first - logger.debug(message) - # Call hidden function. - _log_on_refresh(message) - @abstractproperty def needstickers(self) -> bool: """ @@ -79,13 +60,14 @@ class IPairList(ABC): -> Please overwrite in subclasses """ - def _validate_pair(self, ticker) -> bool: + def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: """ Check one pair against Pairlist Handler's specific conditions. Either implement it in the Pairlist Handler or override the generic filter_pairlist() method. + :param pair: Pair that's currently validated :param ticker: ticker dict as returned from ccxt.load_markets() :return: True if the pair can stay, false if it should be removed """ @@ -128,7 +110,7 @@ class IPairList(ABC): # Copy list since we're modifying this list for p in deepcopy(pairlist): # Filter out assets - if not self._validate_pair(tickers[p]): + if not self._validate_pair(p, tickers[p] if p in tickers else {}): pairlist.remove(p) return pairlist diff --git a/freqtrade/plugins/pairlist/PerformanceFilter.py b/freqtrade/plugins/pairlist/PerformanceFilter.py new file mode 100644 index 000000000..7d91bb77c --- /dev/null +++ b/freqtrade/plugins/pairlist/PerformanceFilter.py @@ -0,0 +1,66 @@ +""" +Performance pair list filter +""" +import logging +from typing import Any, Dict, List + +import pandas as pd + +from freqtrade.persistence import Trade +from freqtrade.plugins.pairlist.IPairList import IPairList + + +logger = logging.getLogger(__name__) + + +class PerformanceFilter(IPairList): + + def __init__(self, exchange, pairlistmanager, + config: Dict[str, Any], pairlistconfig: Dict[str, Any], + pairlist_pos: int) -> None: + super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + + @property + def needstickers(self) -> bool: + """ + Boolean property defining if tickers are necessary. + If no Pairlist requries tickers, an empty List is passed + as tickers argument to filter_pairlist + """ + return False + + def short_desc(self) -> str: + """ + Short allowlist method description - used for startup-messages + """ + return f"{self.name} - Sorting pairs by performance." + + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: + """ + Filters and sorts pairlist and returns the allowlist again. + Called on each bot iteration - please use internal caching if necessary + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + # Get the trading performance for pairs from database + performance = pd.DataFrame(Trade.get_overall_performance()) + + # Skip performance-based sorting if no performance data is available + if len(performance) == 0: + return pairlist + + # Get pairlist from performance dataframe values + list_df = pd.DataFrame({'pair': pairlist}) + + # Set initial value for pairs with no trades to 0 + # Sort the list using: + # - primarily performance (high to low) + # - then count (low to high, so as to favor same performance with fewer trades) + # - then pair name alphametically + sorted_df = list_df.merge(performance, on='pair', how='left')\ + .fillna(0).sort_values(by=['count', 'pair'], ascending=True)\ + .sort_values(by=['profit'], ascending=False) + pairlist = sorted_df['pair'].tolist() + + return pairlist diff --git a/freqtrade/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py similarity index 76% rename from freqtrade/pairlist/PrecisionFilter.py rename to freqtrade/plugins/pairlist/PrecisionFilter.py index 29e32fd44..519337f29 100644 --- a/freqtrade/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -5,7 +5,7 @@ import logging from typing import Any, Dict from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -43,25 +43,25 @@ class PrecisionFilter(IPairList): """ return f"{self.name} - Filtering untradable pairs." - def _validate_pair(self, ticker: dict) -> bool: + def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: """ Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very low value pairs. + :param pair: Pair that's currently validated :param ticker: ticker dict as returned from ccxt.load_markets() - :return: True if the pair can stay, False if it should be removed + :return: True if the pair can stay, false if it should be removed """ stop_price = ticker['ask'] * self._stoploss # Adjust stop-prices to precision - sp = self._exchange.price_to_precision(ticker["symbol"], stop_price) + sp = self._exchange.price_to_precision(pair, stop_price) - stop_gap_price = self._exchange.price_to_precision(ticker["symbol"], stop_price * 0.99) + stop_gap_price = self._exchange.price_to_precision(pair, stop_price * 0.99) logger.debug(f"{ticker['symbol']} - {sp} : {stop_gap_price}") if sp <= stop_gap_price: - self.log_on_refresh(logger.info, - f"Removed {ticker['symbol']} from whitelist, " - f"because stop price {sp} would be <= stop limit {stop_gap_price}") + self.log_once(f"Removed {ticker['symbol']} from whitelist, because " + f"stop price {sp} would be <= stop limit {stop_gap_price}", logger.info) return False return True diff --git a/freqtrade/pairlist/PriceFilter.py b/freqtrade/plugins/pairlist/PriceFilter.py similarity index 75% rename from freqtrade/pairlist/PriceFilter.py rename to freqtrade/plugins/pairlist/PriceFilter.py index bef1c0a15..6558f196f 100644 --- a/freqtrade/pairlist/PriceFilter.py +++ b/freqtrade/plugins/pairlist/PriceFilter.py @@ -5,7 +5,7 @@ import logging from typing import Any, Dict from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -57,39 +57,40 @@ class PriceFilter(IPairList): return f"{self.name} - No price filters configured." - def _validate_pair(self, ticker) -> bool: + def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: """ Check if if one price-step (pip) is > than a certain barrier. + :param pair: Pair that's currently validated :param ticker: ticker dict as returned from ccxt.load_markets() :return: True if the pair can stay, false if it should be removed """ if ticker['last'] is None or ticker['last'] == 0: - self.log_on_refresh(logger.info, - f"Removed {ticker['symbol']} from whitelist, because " - "ticker['last'] is empty (Usually no trade in the last 24h).") + self.log_once(f"Removed {pair} from whitelist, because " + "ticker['last'] is empty (Usually no trade in the last 24h).", + logger.info) return False # Perform low_price_ratio check. if self._low_price_ratio != 0: - compare = self._exchange.price_get_one_pip(ticker['symbol'], ticker['last']) + compare = self._exchange.price_get_one_pip(pair, ticker['last']) changeperc = compare / ticker['last'] if changeperc > self._low_price_ratio: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because 1 unit is {changeperc * 100:.3f}%") + self.log_once(f"Removed {pair} from whitelist, " + f"because 1 unit is {changeperc * 100:.3f}%", logger.info) return False # Perform min_price check. if self._min_price != 0: if ticker['last'] < self._min_price: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because last price < {self._min_price:.8f}") + self.log_once(f"Removed {pair} from whitelist, " + f"because last price < {self._min_price:.8f}", logger.info) return False # Perform max_price check. if self._max_price != 0: if ticker['last'] > self._max_price: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because last price > {self._max_price:.8f}") + self.log_once(f"Removed {ticker['symbol']} from whitelist, " + f"because last price > {self._max_price:.8f}", logger.info) return False return True diff --git a/freqtrade/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py similarity index 96% rename from freqtrade/pairlist/ShuffleFilter.py rename to freqtrade/plugins/pairlist/ShuffleFilter.py index 28778db7b..4d3dd29e3 100644 --- a/freqtrade/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -5,7 +5,7 @@ import logging import random from typing import Any, Dict, List -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) diff --git a/freqtrade/pairlist/SpreadFilter.py b/freqtrade/plugins/pairlist/SpreadFilter.py similarity index 75% rename from freqtrade/pairlist/SpreadFilter.py rename to freqtrade/plugins/pairlist/SpreadFilter.py index a636b90bd..2f3fe47e3 100644 --- a/freqtrade/pairlist/SpreadFilter.py +++ b/freqtrade/plugins/pairlist/SpreadFilter.py @@ -4,7 +4,7 @@ Spread pair list filter import logging from typing import Any, Dict -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -36,18 +36,19 @@ class SpreadFilter(IPairList): return (f"{self.name} - Filtering pairs with ask/bid diff above " f"{self._max_spread_ratio * 100}%.") - def _validate_pair(self, ticker: dict) -> bool: + def _validate_pair(self, pair: str, ticker: Dict[str, Any]) -> bool: """ Validate spread for the ticker + :param pair: Pair that's currently validated :param ticker: ticker dict as returned from ccxt.load_markets() - :return: True if the pair can stay, False if it should be removed + :return: True if the pair can stay, false if it should be removed """ if 'bid' in ticker and 'ask' in ticker: spread = 1 - ticker['bid'] / ticker['ask'] if spread > self._max_spread_ratio: - self.log_on_refresh(logger.info, f"Removed {ticker['symbol']} from whitelist, " - f"because spread {spread * 100:.3f}% >" - f"{self._max_spread_ratio * 100}%") + self.log_once(f"Removed {pair} from whitelist, because spread " + f"{spread * 100:.3f}% > {self._max_spread_ratio * 100}%", + logger.info) return False else: return True diff --git a/freqtrade/pairlist/StaticPairList.py b/freqtrade/plugins/pairlist/StaticPairList.py similarity index 97% rename from freqtrade/pairlist/StaticPairList.py rename to freqtrade/plugins/pairlist/StaticPairList.py index 2879cb364..dd592e0ca 100644 --- a/freqtrade/pairlist/StaticPairList.py +++ b/freqtrade/plugins/pairlist/StaticPairList.py @@ -7,7 +7,7 @@ import logging from typing import Any, Dict, List from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) diff --git a/freqtrade/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py similarity index 96% rename from freqtrade/pairlist/VolumePairList.py rename to freqtrade/plugins/pairlist/VolumePairList.py index 7d3c2c653..dd8fc64fd 100644 --- a/freqtrade/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -8,7 +8,7 @@ from datetime import datetime from typing import Any, Dict, List from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -111,6 +111,6 @@ class VolumePairList(IPairList): # Limit pairlist to the requested number of pairs pairs = pairs[:self._number_pairs] - self.log_on_refresh(logger.info, f"Searching {self._number_pairs} pairs: {pairs}") + self.log_once(f"Searching {self._number_pairs} pairs: {pairs}", logger.info) return pairs diff --git a/tests/pairlist/__init__.py b/freqtrade/plugins/pairlist/__init__.py similarity index 100% rename from tests/pairlist/__init__.py rename to freqtrade/plugins/pairlist/__init__.py diff --git a/freqtrade/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py similarity index 61% rename from freqtrade/pairlist/rangestabilityfilter.py rename to freqtrade/plugins/pairlist/rangestabilityfilter.py index b460ff477..f2e84930b 100644 --- a/freqtrade/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -2,14 +2,16 @@ Rate of change pairlist filter """ import logging -from typing import Any, Dict +from copy import deepcopy +from typing import Any, Dict, List, Optional import arrow from cachetools.ttl import TTLCache +from pandas import DataFrame from freqtrade.exceptions import OperationalException from freqtrade.misc import plural -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList logger = logging.getLogger(__name__) @@ -42,7 +44,7 @@ class RangeStabilityFilter(IPairList): If no Pairlist requires tickers, an empty List is passed as tickers argument to filter_pairlist """ - return True + return False def short_desc(self) -> str: """ @@ -51,25 +53,43 @@ class RangeStabilityFilter(IPairList): return (f"{self.name} - Filtering pairs with rate of change below " f"{self._min_rate_of_change} over the last {plural(self._days, 'day')}.") - def _validate_pair(self, ticker: Dict) -> bool: + def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]: """ Validate trading range - :param ticker: ticker dict as returned from ccxt.load_markets() - :return: True if the pair can stay, False if it should be removed + :param pairlist: pairlist to filter or sort + :param tickers: Tickers (from exchange.get_tickers()). May be cached. + :return: new allowlist + """ + needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] + + since_ms = int(arrow.utcnow() + .floor('day') + .shift(days=-self._days - 1) + .float_timestamp) * 1000 + # Get all candles + candles = {} + if needed_pairs: + candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, + cache=False) + + if self._enabled: + for p in deepcopy(pairlist): + daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None + if not self._validate_pair_loc(p, daily_candles): + pairlist.remove(p) + return pairlist + + def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool: + """ + Validate trading range + :param pair: Pair that's currently validated + :param ticker: ticker dict as returned from ccxt.load_markets() + :return: True if the pair can stay, false if it should be removed """ - pair = ticker['symbol'] # Check symbol in cache if pair in self._pair_cache: return self._pair_cache[pair] - since_ms = int(arrow.utcnow() - .floor('day') - .shift(days=-self._days) - .float_timestamp) * 1000 - - daily_candles = self._exchange.get_historic_ohlcv_as_df(pair=pair, - timeframe='1d', - since_ms=since_ms) result = False if daily_candles is not None and not daily_candles.empty: highest_high = daily_candles['high'].max() @@ -78,11 +98,10 @@ class RangeStabilityFilter(IPairList): if pct_change >= self._min_rate_of_change: result = True else: - self.log_on_refresh(logger.info, - f"Removed {pair} from whitelist, " - f"because rate of change over {plural(self._days, 'day')} is " - f"{pct_change:.3f}, which is below the " - f"threshold of {self._min_rate_of_change}.") + self.log_once(f"Removed {pair} from whitelist, because rate of change " + f"over {self._days} {plural(self._days, 'day')} is {pct_change:.3f}, " + f"which is below the threshold of {self._min_rate_of_change}.", + logger.info) result = False self._pair_cache[pair] = result diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py similarity index 93% rename from freqtrade/pairlist/pairlistmanager.py rename to freqtrade/plugins/pairlistmanager.py index 89bab99be..b71f02898 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -3,13 +3,13 @@ PairList manager class """ import logging from copy import deepcopy -from typing import Dict, List +from typing import Any, Dict, List from cachetools import TTLCache, cached from freqtrade.constants import ListPairsWithTimeframes from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.resolvers import PairListResolver @@ -26,9 +26,6 @@ class PairListManager(): self._pairlist_handlers: List[IPairList] = [] self._tickers_needed = False for pairlist_handler_config in self._config.get('pairlists', None): - if 'method' not in pairlist_handler_config: - logger.warning(f"No method found in {pairlist_handler_config}, ignoring.") - continue pairlist_handler = PairListResolver.load_pairlist( pairlist_handler_config['method'], exchange=exchange, @@ -100,7 +97,7 @@ class PairListManager(): self._whitelist = pairlist - def _prepare_whitelist(self, pairlist: List[str], tickers) -> List[str]: + def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]: """ Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove pairs that do not have ticker available diff --git a/freqtrade/plugins/protectionmanager.py b/freqtrade/plugins/protectionmanager.py new file mode 100644 index 000000000..a8edd4e4b --- /dev/null +++ b/freqtrade/plugins/protectionmanager.py @@ -0,0 +1,72 @@ +""" +Protection manager class +""" +import logging +from datetime import datetime, timezone +from typing import Dict, List, Optional + +from freqtrade.persistence import PairLocks +from freqtrade.plugins.protections import IProtection +from freqtrade.resolvers import ProtectionResolver + + +logger = logging.getLogger(__name__) + + +class ProtectionManager(): + + def __init__(self, config: dict) -> None: + self._config = config + + self._protection_handlers: List[IProtection] = [] + for protection_handler_config in self._config.get('protections', []): + protection_handler = ProtectionResolver.load_protection( + protection_handler_config['method'], + config=config, + protection_config=protection_handler_config, + ) + self._protection_handlers.append(protection_handler) + + if not self._protection_handlers: + logger.info("No protection Handlers defined.") + + @property + def name_list(self) -> List[str]: + """ + Get list of loaded Protection Handler names + """ + return [p.name for p in self._protection_handlers] + + def short_desc(self) -> List[Dict]: + """ + List of short_desc for each Pairlist Handler + """ + return [{p.name: p.short_desc()} for p in self._protection_handlers] + + def global_stop(self, now: Optional[datetime] = None) -> bool: + if not now: + now = datetime.now(timezone.utc) + result = False + for protection_handler in self._protection_handlers: + if protection_handler.has_global_stop: + result, until, reason = protection_handler.global_stop(now) + + # Early stopping - first positive result blocks further trades + if result and until: + if not PairLocks.is_global_lock(until): + PairLocks.lock_pair('*', until, reason, now=now) + result = True + return result + + def stop_per_pair(self, pair, now: Optional[datetime] = None) -> bool: + if not now: + now = datetime.now(timezone.utc) + result = False + for protection_handler in self._protection_handlers: + if protection_handler.has_local_stop: + result, until, reason = protection_handler.stop_per_pair(pair, now) + if result and until: + if not PairLocks.is_pair_locked(pair, until): + PairLocks.lock_pair(pair, until, reason, now=now) + result = True + return result diff --git a/freqtrade/plugins/protections/__init__.py b/freqtrade/plugins/protections/__init__.py new file mode 100644 index 000000000..936355052 --- /dev/null +++ b/freqtrade/plugins/protections/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa: F401 +from freqtrade.plugins.protections.iprotection import IProtection, ProtectionReturn diff --git a/freqtrade/plugins/protections/cooldown_period.py b/freqtrade/plugins/protections/cooldown_period.py new file mode 100644 index 000000000..2d7d7b4c7 --- /dev/null +++ b/freqtrade/plugins/protections/cooldown_period.py @@ -0,0 +1,72 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn + + +logger = logging.getLogger(__name__) + + +class CooldownPeriod(IProtection): + + has_global_stop: bool = False + has_local_stop: bool = True + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + def _reason(self) -> str: + """ + LockReason to use + """ + return (f'Cooldown period for {self.stop_duration_str}.') + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Cooldown period of {self.stop_duration_str}.") + + def _cooldown_period(self, pair: str, date_now: datetime, ) -> ProtectionReturn: + """ + Get last trade for this pair + """ + look_back_until = date_now - timedelta(minutes=self._stop_duration) + # filters = [ + # Trade.is_open.is_(False), + # Trade.close_date > look_back_until, + # Trade.pair == pair, + # ] + # trade = Trade.get_trades(filters).first() + trades = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) + if trades: + # Get latest trade + trade = sorted(trades, key=lambda t: t.close_date)[-1] + self.log_once(f"Cooldown for {pair} for {self.stop_duration_str}.", logger.info) + until = self.calculate_lock_end([trade], self._stop_duration) + + return True, until, self._reason() + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + # Not implemented for cooldown period. + return False, None, None + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return self._cooldown_period(pair, date_now) diff --git a/freqtrade/plugins/protections/iprotection.py b/freqtrade/plugins/protections/iprotection.py new file mode 100644 index 000000000..684bf6cd3 --- /dev/null +++ b/freqtrade/plugins/protections/iprotection.py @@ -0,0 +1,107 @@ + +import logging +from abc import ABC, abstractmethod +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Tuple + +from freqtrade.exchange import timeframe_to_minutes +from freqtrade.misc import plural +from freqtrade.mixins import LoggingMixin +from freqtrade.persistence import Trade + + +logger = logging.getLogger(__name__) + +ProtectionReturn = Tuple[bool, Optional[datetime], Optional[str]] + + +class IProtection(LoggingMixin, ABC): + + # Can globally stop the bot + has_global_stop: bool = False + # Can stop trading for one pair + has_local_stop: bool = False + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + self._config = config + self._protection_config = protection_config + tf_in_min = timeframe_to_minutes(config['timeframe']) + if 'stop_duration_candles' in protection_config: + self._stop_duration_candles = protection_config.get('stop_duration_candles', 1) + self._stop_duration = (tf_in_min * self._stop_duration_candles) + else: + self._stop_duration_candles = None + self._stop_duration = protection_config.get('stop_duration', 60) + if 'lookback_period_candles' in protection_config: + self._lookback_period_candles = protection_config.get('lookback_period_candles', 1) + self._lookback_period = tf_in_min * self._lookback_period_candles + else: + self._lookback_period_candles = None + self._lookback_period = protection_config.get('lookback_period', 60) + + LoggingMixin.__init__(self, logger) + + @property + def name(self) -> str: + return self.__class__.__name__ + + @property + def stop_duration_str(self) -> str: + """ + Output configured stop duration in either candles or minutes + """ + if self._stop_duration_candles: + return (f"{self._stop_duration_candles} " + f"{plural(self._stop_duration_candles, 'candle', 'candles')}") + else: + return (f"{self._stop_duration} " + f"{plural(self._stop_duration, 'minute', 'minutes')}") + + @property + def lookback_period_str(self) -> str: + """ + Output configured lookback period in either candles or minutes + """ + if self._lookback_period_candles: + return (f"{self._lookback_period_candles} " + f"{plural(self._lookback_period_candles, 'candle', 'candles')}") + else: + return (f"{self._lookback_period} " + f"{plural(self._lookback_period, 'minute', 'minutes')}") + + @abstractmethod + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + -> Please overwrite in subclasses + """ + + @abstractmethod + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + """ + + @abstractmethod + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + + @staticmethod + def calculate_lock_end(trades: List[Trade], stop_minutes: int) -> datetime: + """ + Get lock end time + """ + max_date: datetime = max([trade.close_date for trade in trades]) + # comming from Database, tzinfo is not set. + if max_date.tzinfo is None: + max_date = max_date.replace(tzinfo=timezone.utc) + + until = max_date + timedelta(minutes=stop_minutes) + + return until diff --git a/freqtrade/plugins/protections/low_profit_pairs.py b/freqtrade/plugins/protections/low_profit_pairs.py new file mode 100644 index 000000000..9d5ed35b4 --- /dev/null +++ b/freqtrade/plugins/protections/low_profit_pairs.py @@ -0,0 +1,83 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn + + +logger = logging.getLogger(__name__) + + +class LowProfitPairs(IProtection): + + has_global_stop: bool = False + has_local_stop: bool = True + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + self._trade_limit = protection_config.get('trade_limit', 1) + self._required_profit = protection_config.get('required_profit', 0.0) + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Low Profit Protection, locks pairs with " + f"profit < {self._required_profit} within {self.lookback_period_str}.") + + def _reason(self, profit: float) -> str: + """ + LockReason to use + """ + return (f'{profit} < {self._required_profit} in {self.lookback_period_str}, ' + f'locking for {self.stop_duration_str}.') + + def _low_profit(self, date_now: datetime, pair: str) -> ProtectionReturn: + """ + Evaluate recent trades for pair + """ + look_back_until = date_now - timedelta(minutes=self._lookback_period) + # filters = [ + # Trade.is_open.is_(False), + # Trade.close_date > look_back_until, + # ] + # if pair: + # filters.append(Trade.pair == pair) + + trades = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) + # trades = Trade.get_trades(filters).all() + if len(trades) < self._trade_limit: + # Not enough trades in the relevant period + return False, None, None + + profit = sum(trade.close_profit for trade in trades) + if profit < self._required_profit: + self.log_once( + f"Trading for {pair} stopped due to {profit:.2f} < {self._required_profit} " + f"within {self._lookback_period} minutes.", logger.info) + until = self.calculate_lock_end(trades, self._stop_duration) + + return True, until, self._reason(profit) + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + return False, None, None + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return self._low_profit(date_now, pair=pair) diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py new file mode 100644 index 000000000..d54e6699b --- /dev/null +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -0,0 +1,88 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +import pandas as pd + +from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn + + +logger = logging.getLogger(__name__) + + +class MaxDrawdown(IProtection): + + has_global_stop: bool = True + has_local_stop: bool = False + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + self._trade_limit = protection_config.get('trade_limit', 1) + self._max_allowed_drawdown = protection_config.get('max_allowed_drawdown', 0.0) + # TODO: Implement checks to limit max_drawdown to sensible values + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Max drawdown protection, stop trading if drawdown is > " + f"{self._max_allowed_drawdown} within {self.lookback_period_str}.") + + def _reason(self, drawdown: float) -> str: + """ + LockReason to use + """ + return (f'{drawdown} > {self._max_allowed_drawdown} in {self.lookback_period_str}, ' + f'locking for {self.stop_duration_str}.') + + def _max_drawdown(self, date_now: datetime) -> ProtectionReturn: + """ + Evaluate recent trades for drawdown ... + """ + look_back_until = date_now - timedelta(minutes=self._lookback_period) + + trades = Trade.get_trades_proxy(is_open=False, close_date=look_back_until) + + trades_df = pd.DataFrame([trade.to_json() for trade in trades]) + + if len(trades) < self._trade_limit: + # Not enough trades in the relevant period + return False, None, None + + # Drawdown is always positive + try: + drawdown, _, _ = calculate_max_drawdown(trades_df, value_col='close_profit') + except ValueError: + return False, None, None + + if drawdown > self._max_allowed_drawdown: + self.log_once( + f"Trading stopped due to Max Drawdown {drawdown:.2f} < {self._max_allowed_drawdown}" + f" within {self.lookback_period_str}.", logger.info) + until = self.calculate_lock_end(trades, self._stop_duration) + + return True, until, self._reason(drawdown) + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + return self._max_drawdown(date_now) + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return False, None, None diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py new file mode 100644 index 000000000..193907ddc --- /dev/null +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -0,0 +1,86 @@ + +import logging +from datetime import datetime, timedelta +from typing import Any, Dict + +from freqtrade.persistence import Trade +from freqtrade.plugins.protections import IProtection, ProtectionReturn +from freqtrade.strategy.interface import SellType + + +logger = logging.getLogger(__name__) + + +class StoplossGuard(IProtection): + + has_global_stop: bool = True + has_local_stop: bool = True + + def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None: + super().__init__(config, protection_config) + + self._trade_limit = protection_config.get('trade_limit', 10) + self._disable_global_stop = protection_config.get('only_per_pair', False) + + def short_desc(self) -> str: + """ + Short method description - used for startup-messages + """ + return (f"{self.name} - Frequent Stoploss Guard, {self._trade_limit} stoplosses " + f"within {self.lookback_period_str}.") + + def _reason(self) -> str: + """ + LockReason to use + """ + return (f'{self._trade_limit} stoplosses in {self._lookback_period} min, ' + f'locking for {self._stop_duration} min.') + + def _stoploss_guard(self, date_now: datetime, pair: str = None) -> ProtectionReturn: + """ + Evaluate recent trades + """ + look_back_until = date_now - timedelta(minutes=self._lookback_period) + # filters = [ + # Trade.is_open.is_(False), + # Trade.close_date > look_back_until, + # or_(Trade.sell_reason == SellType.STOP_LOSS.value, + # and_(Trade.sell_reason == SellType.TRAILING_STOP_LOSS.value, + # Trade.close_profit < 0)) + # ] + # if pair: + # filters.append(Trade.pair == pair) + # trades = Trade.get_trades(filters).all() + + trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) + trades = [trade for trade in trades1 if str(trade.sell_reason) == SellType.STOP_LOSS.value + or (str(trade.sell_reason) == SellType.TRAILING_STOP_LOSS.value + and trade.close_profit < 0)] + + if len(trades) > self._trade_limit: + self.log_once(f"Trading stopped due to {self._trade_limit} " + f"stoplosses within {self._lookback_period} minutes.", logger.info) + until = self.calculate_lock_end(trades, self._stop_duration) + return True, until, self._reason() + + return False, None, None + + def global_stop(self, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for all pairs + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, all pairs will be locked with until + """ + if self._disable_global_stop: + return False, None, None + return self._stoploss_guard(date_now, None) + + def stop_per_pair(self, pair: str, date_now: datetime) -> ProtectionReturn: + """ + Stops trading (position entering) for this pair + This must evaluate to true for the whole period of the "cooldown period". + :return: Tuple of [bool, until, reason]. + If true, this pair will be locked with until + """ + return self._stoploss_guard(date_now, pair) diff --git a/freqtrade/resolvers/__init__.py b/freqtrade/resolvers/__init__.py index b42ec4931..ef24bf481 100644 --- a/freqtrade/resolvers/__init__.py +++ b/freqtrade/resolvers/__init__.py @@ -6,6 +6,7 @@ from freqtrade.resolvers.exchange_resolver import ExchangeResolver # Don't import HyperoptResolver to avoid loading the whole Optimize tree # from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.resolvers.pairlist_resolver import PairListResolver +from freqtrade.resolvers.protection_resolver import ProtectionResolver from freqtrade.resolvers.strategy_resolver import StrategyResolver diff --git a/freqtrade/resolvers/pairlist_resolver.py b/freqtrade/resolvers/pairlist_resolver.py index 4df5da37c..72a3cc1dd 100644 --- a/freqtrade/resolvers/pairlist_resolver.py +++ b/freqtrade/resolvers/pairlist_resolver.py @@ -6,7 +6,7 @@ This module load custom pairlists import logging from pathlib import Path -from freqtrade.pairlist.IPairList import IPairList +from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.resolvers import IResolver @@ -20,7 +20,7 @@ class PairListResolver(IResolver): object_type = IPairList object_type_str = "Pairlist" user_subdir = None - initial_search_path = Path(__file__).parent.parent.joinpath('pairlist').resolve() + initial_search_path = Path(__file__).parent.parent.joinpath('plugins/pairlist').resolve() @staticmethod def load_pairlist(pairlist_name: str, exchange, pairlistmanager, diff --git a/freqtrade/resolvers/protection_resolver.py b/freqtrade/resolvers/protection_resolver.py new file mode 100644 index 000000000..c54ae1011 --- /dev/null +++ b/freqtrade/resolvers/protection_resolver.py @@ -0,0 +1,37 @@ +""" +This module load custom pairlists +""" +import logging +from pathlib import Path +from typing import Dict + +from freqtrade.plugins.protections import IProtection +from freqtrade.resolvers import IResolver + + +logger = logging.getLogger(__name__) + + +class ProtectionResolver(IResolver): + """ + This class contains all the logic to load custom PairList class + """ + object_type = IProtection + object_type_str = "Protection" + user_subdir = None + initial_search_path = Path(__file__).parent.parent.joinpath('plugins/protections').resolve() + + @staticmethod + def load_protection(protection_name: str, config: Dict, protection_config: Dict) -> IProtection: + """ + Load the protection with protection_name + :param protection_name: Classname of the pairlist + :param config: configuration dictionary + :param protection_config: Configuration dedicated to this pairlist + :return: initialized Protection class + """ + return ProtectionResolver.load_object(protection_name, config, + kwargs={'config': config, + 'protection_config': protection_config, + }, + ) diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 63a3f784e..73af00fee 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -88,9 +88,6 @@ class StrategyResolver(IResolver): StrategyResolver._override_attribute_helper(strategy, config, attribute, default) - # Assign deprecated variable - to not break users code relying on this. - strategy.ticker_interval = strategy.timeframe - # Loop this list again to have output combined for attribute, _, subkey in attributes: if subkey and attribute in config[subkey]: @@ -98,11 +95,7 @@ class StrategyResolver(IResolver): elif attribute in config: logger.info("Strategy using %s: %s", attribute, config[attribute]) - # Sort and apply type conversions - strategy.minimal_roi = OrderedDict(sorted( - {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), - key=lambda t: t[0])) - strategy.stoploss = float(strategy.stoploss) + StrategyResolver._normalize_attributes(strategy) StrategyResolver._strategy_sanity_validations(strategy) return strategy @@ -131,6 +124,24 @@ class StrategyResolver(IResolver): setattr(strategy, attribute, default) config[attribute] = default + @staticmethod + def _normalize_attributes(strategy: IStrategy) -> IStrategy: + """ + Normalize attributes to have the correct type. + """ + # Assign deprecated variable - to not break users code relying on this. + if hasattr(strategy, 'timeframe'): + strategy.ticker_interval = strategy.timeframe + + # Sort and apply type conversions + if hasattr(strategy, 'minimal_roi'): + strategy.minimal_roi = OrderedDict(sorted( + {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), + key=lambda t: t[0])) + if hasattr(strategy, 'stoploss'): + strategy.stoploss = float(strategy.stoploss) + return strategy + @staticmethod def _strategy_sanity_validations(strategy): if not all(k in strategy.order_types for k in REQUIRED_ORDERTYPES): diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index 88978519b..0a0130ca7 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -1,3 +1,3 @@ # flake8: noqa: F401 -from .rpc import RPC, RPCException, RPCMessageType +from .rpc import RPC, RPCException, RPCHandler, RPCMessageType from .rpc_manager import RPCManager diff --git a/freqtrade/rpc/api_server.py b/freqtrade/rpc/api_server.py index 384d7c6c2..b489586c8 100644 --- a/freqtrade/rpc/api_server.py +++ b/freqtrade/rpc/api_server.py @@ -20,8 +20,7 @@ from freqtrade.__init__ import __version__ from freqtrade.constants import DATETIME_PRINT_FORMAT, USERPATH_STRATEGIES from freqtrade.exceptions import OperationalException from freqtrade.persistence import Trade -from freqtrade.rpc.fiat_convert import CryptoToFiatConverter -from freqtrade.rpc.rpc import RPC, RPCException +from freqtrade.rpc.rpc import RPC, RPCException, RPCHandler logger = logging.getLogger(__name__) @@ -79,7 +78,7 @@ def shutdown_session(exception=None): Trade.session.remove() -class ApiServer(RPC): +class ApiServer(RPCHandler): """ This class runs api server and provides rpc.rpc functionality to it @@ -90,15 +89,15 @@ class ApiServer(RPC): return (safe_str_cmp(username, self._config['api_server'].get('username')) and safe_str_cmp(password, self._config['api_server'].get('password'))) - def __init__(self, freqtrade) -> None: + def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: """ - Init the api server, and init the super class RPC - :param freqtrade: Instance of a freqtrade bot + Init the api server, and init the super class RPCHandler + :param rpc: instance of RPC Helper class + :param config: Configuration object :return: None """ - super().__init__(freqtrade) + super().__init__(rpc, config) - self._config = freqtrade.config self.app = Flask(__name__) self._cors = CORS(self.app, resources={r"/api/*": { @@ -118,9 +117,6 @@ class ApiServer(RPC): # Register application handling self.register_rest_rpc_urls() - if self._config.get('fiat_display_currency', None): - self._fiat_converter = CryptoToFiatConverter() - thread = threading.Thread(target=self.run, daemon=True) thread.start() @@ -198,6 +194,8 @@ class ApiServer(RPC): self.app.add_url_rule(f'{BASE_URI}/logs', 'log', view_func=self._get_logs, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/profit', 'profit', view_func=self._profit, methods=['GET']) + self.app.add_url_rule(f'{BASE_URI}/stats', 'stats', + view_func=self._stats, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/performance', 'performance', view_func=self._performance, methods=['GET']) self.app.add_url_rule(f'{BASE_URI}/status', 'status', @@ -285,7 +283,7 @@ class ApiServer(RPC): Handler for /start. Starts TradeThread in bot if stopped. """ - msg = self._rpc_start() + msg = self._rpc._rpc_start() return jsonify(msg) @require_login @@ -295,7 +293,7 @@ class ApiServer(RPC): Handler for /stop. Stops TradeThread in bot if running """ - msg = self._rpc_stop() + msg = self._rpc._rpc_stop() return jsonify(msg) @require_login @@ -305,7 +303,7 @@ class ApiServer(RPC): Handler for /stopbuy. Sets max_open_trades to 0 and gracefully sells all open trades """ - msg = self._rpc_stopbuy() + msg = self._rpc._rpc_stopbuy() return jsonify(msg) @rpc_catch_errors @@ -329,7 +327,7 @@ class ApiServer(RPC): """ Prints the bot's version """ - return jsonify(RPC._rpc_show_config(self._config, self._freqtrade.state)) + return jsonify(RPC._rpc_show_config(self._config, self._rpc._freqtrade.state)) @require_login @rpc_catch_errors @@ -338,7 +336,7 @@ class ApiServer(RPC): Handler for /reload_config. Triggers a config file reload """ - msg = self._rpc_reload_config() + msg = self._rpc._rpc_reload_config() return jsonify(msg) @require_login @@ -348,7 +346,7 @@ class ApiServer(RPC): Handler for /count. Returns the number of trades running """ - msg = self._rpc_count() + msg = self._rpc._rpc_count() return jsonify(msg) @require_login @@ -358,7 +356,7 @@ class ApiServer(RPC): Handler for /locks. Returns the currently active locks. """ - return jsonify(self._rpc_locks()) + return jsonify(self._rpc._rpc_locks()) @require_login @rpc_catch_errors @@ -371,10 +369,10 @@ class ApiServer(RPC): timescale = request.args.get('timescale', 7) timescale = int(timescale) - stats = self._rpc_daily_profit(timescale, - self._config['stake_currency'], - self._config.get('fiat_display_currency', '') - ) + stats = self._rpc._rpc_daily_profit(timescale, + self._config['stake_currency'], + self._config.get('fiat_display_currency', '') + ) return jsonify(stats) @@ -388,7 +386,7 @@ class ApiServer(RPC): limit: Only get a certain number of records """ limit = int(request.args.get('limit', 0)) or None - return jsonify(self._rpc_get_logs(limit)) + return jsonify(RPC._rpc_get_logs(limit)) @require_login @rpc_catch_errors @@ -397,7 +395,7 @@ class ApiServer(RPC): Returns information related to Edge. :return: edge stats """ - stats = self._rpc_edge() + stats = self._rpc._rpc_edge() return jsonify(stats) @@ -411,9 +409,21 @@ class ApiServer(RPC): :return: stats """ - stats = self._rpc_trade_statistics(self._config['stake_currency'], - self._config.get('fiat_display_currency') - ) + stats = self._rpc._rpc_trade_statistics(self._config['stake_currency'], + self._config.get('fiat_display_currency') + ) + + return jsonify(stats) + + @require_login + @rpc_catch_errors + def _stats(self): + """ + Handler for /stats. + Returns a Object with "durations" and "sell_reasons" as keys. + """ + + stats = self._rpc._rpc_stats() return jsonify(stats) @@ -426,7 +436,7 @@ class ApiServer(RPC): Returns a cumulative performance statistics :return: stats """ - stats = self._rpc_performance() + stats = self._rpc._rpc_performance() return jsonify(stats) @@ -439,7 +449,7 @@ class ApiServer(RPC): Returns the current status of the trades in json format """ try: - results = self._rpc_trade_status() + results = self._rpc._rpc_trade_status() return jsonify(results) except RPCException: return jsonify([]) @@ -452,8 +462,8 @@ class ApiServer(RPC): Returns the current status of the trades in json format """ - results = self._rpc_balance(self._config['stake_currency'], - self._config.get('fiat_display_currency', '')) + results = self._rpc._rpc_balance(self._config['stake_currency'], + self._config.get('fiat_display_currency', '')) return jsonify(results) @require_login @@ -465,12 +475,12 @@ class ApiServer(RPC): Returns the X last trades in json format """ limit = int(request.args.get('limit', 0)) - results = self._rpc_trade_history(limit) + results = self._rpc._rpc_trade_history(limit) return jsonify(results) @require_login @rpc_catch_errors - def _trades_delete(self, tradeid): + def _trades_delete(self, tradeid: int): """ Handler for DELETE /trades/ endpoint. Removes the trade from the database (tries to cancel open orders first!) @@ -478,7 +488,7 @@ class ApiServer(RPC): param: tradeid: Numeric trade-id assigned to the trade. """ - result = self._rpc_delete(tradeid) + result = self._rpc._rpc_delete(tradeid) return jsonify(result) @require_login @@ -487,7 +497,7 @@ class ApiServer(RPC): """ Handler for /whitelist. """ - results = self._rpc_whitelist() + results = self._rpc._rpc_whitelist() return jsonify(results) @require_login @@ -497,7 +507,7 @@ class ApiServer(RPC): Handler for /blacklist. """ add = request.json.get("blacklist", None) if request.method == 'POST' else None - results = self._rpc_blacklist(add) + results = self._rpc._rpc_blacklist(add) return jsonify(results) @require_login @@ -510,7 +520,7 @@ class ApiServer(RPC): price = request.json.get("price", None) price = float(price) if price is not None else price - trade = self._rpc_forcebuy(asset, price) + trade = self._rpc._rpc_forcebuy(asset, price) if trade: return jsonify(trade.to_json()) else: @@ -523,7 +533,7 @@ class ApiServer(RPC): Handler for /forcesell. """ tradeid = request.json.get("tradeid") - results = self._rpc_forcesell(tradeid) + results = self._rpc._rpc_forcesell(tradeid) return jsonify(results) @require_login @@ -545,7 +555,7 @@ class ApiServer(RPC): if not pair or not timeframe: return self.rest_error("Mandatory parameter missing.", 400) - results = self._rpc_analysed_dataframe(pair, timeframe, limit) + results = self._rpc._rpc_analysed_dataframe(pair, timeframe, limit) return jsonify(results) @require_login @@ -584,7 +594,7 @@ class ApiServer(RPC): """ Handler for /plot_config. """ - return jsonify(self._rpc_plot_config()) + return jsonify(self._rpc._rpc_plot_config()) @require_login @rpc_catch_errors diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index e608a2274..42ab76622 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -65,20 +65,17 @@ class RPCException(Exception): } -class RPC: - """ - RPC class can be used to have extra feature, like bot data, and access to DB data - """ - # Bind _fiat_converter if needed in each RPC handler - _fiat_converter: Optional[CryptoToFiatConverter] = None +class RPCHandler: - def __init__(self, freqtrade) -> None: + def __init__(self, rpc: 'RPC', config: Dict[str, Any]) -> None: """ - Initializes all enabled rpc modules - :param freqtrade: Instance of a freqtrade bot + Initializes RPCHandlers + :param rpc: instance of RPC Helper class + :param config: Configuration object :return: None """ - self._freqtrade = freqtrade + self._rpc = rpc + self._config: Dict[str, Any] = config @property def name(self) -> str: @@ -93,6 +90,25 @@ class RPC: def send_msg(self, msg: Dict[str, str]) -> None: """ Sends a message to all registered rpc modules """ + +class RPC: + """ + RPC class can be used to have extra feature, like bot data, and access to DB data + """ + # Bind _fiat_converter if needed + _fiat_converter: Optional[CryptoToFiatConverter] = None + + def __init__(self, freqtrade) -> None: + """ + Initializes all enabled rpc modules + :param freqtrade: Instance of a freqtrade bot + :return: None + """ + self._freqtrade = freqtrade + self._config: Dict[str, Any] = freqtrade.config + if self._config.get('fiat_display_currency', None): + self._fiat_converter = CryptoToFiatConverter() + @staticmethod def _rpc_show_config(config, botstate: State) -> Dict[str, Any]: """ @@ -275,6 +291,39 @@ class RPC: "trades_count": len(output) } + def _rpc_stats(self) -> Dict[str, Any]: + """ + Generate generic stats for trades in database + """ + def trade_win_loss(trade): + if trade.close_profit > 0: + return 'wins' + elif trade.close_profit < 0: + return 'losses' + else: + return 'draws' + trades = trades = Trade.get_trades([Trade.is_open.is_(False)]) + # Sell reason + sell_reasons = {} + for trade in trades: + if trade.sell_reason not in sell_reasons: + sell_reasons[trade.sell_reason] = {'wins': 0, 'losses': 0, 'draws': 0} + sell_reasons[trade.sell_reason][trade_win_loss(trade)] += 1 + + # Duration + dur: Dict[str, List[int]] = {'wins': [], 'draws': [], 'losses': []} + for trade in trades: + if trade.close_date is not None and trade.open_date is not None: + trade_dur = (trade.close_date - trade.open_date).total_seconds() + dur[trade_win_loss(trade)].append(trade_dur) + + wins_dur = sum(dur['wins']) / len(dur['wins']) if len(dur['wins']) > 0 else 'N/A' + draws_dur = sum(dur['draws']) / len(dur['draws']) if len(dur['draws']) > 0 else 'N/A' + losses_dur = sum(dur['losses']) / len(dur['losses']) if len(dur['losses']) > 0 else 'N/A' + + durations = {'wins': wins_dur, 'draws': draws_dur, 'losses': losses_dur} + return {'sell_reasons': sell_reasons, 'durations': durations} + def _rpc_trade_statistics( self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: """ Returns cumulative profit statistics """ @@ -542,7 +591,7 @@ class RPC: else: return None - def _rpc_delete(self, trade_id: str) -> Dict[str, Union[str, int]]: + def _rpc_delete(self, trade_id: int) -> Dict[str, Union[str, int]]: """ Handler for delete . Delete the given trade and close eventually existing open orders. @@ -645,7 +694,8 @@ class RPC: } return res - def _rpc_get_logs(self, limit: Optional[int]) -> Dict[str, Any]: + @staticmethod + def _rpc_get_logs(limit: Optional[int]) -> Dict[str, Any]: """Returns the last X logs""" if limit: buffer = bufferHandler.buffer[-limit:] diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index b97a5357b..38a4e95fd 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -4,7 +4,7 @@ This module contains class to manage RPC communications (Telegram, Slack, ...) import logging from typing import Any, Dict, List -from freqtrade.rpc import RPC, RPCMessageType +from freqtrade.rpc import RPC, RPCHandler, RPCMessageType logger = logging.getLogger(__name__) @@ -16,25 +16,26 @@ class RPCManager: """ def __init__(self, freqtrade) -> None: """ Initializes all enabled rpc modules """ - self.registered_modules: List[RPC] = [] - + self.registered_modules: List[RPCHandler] = [] + self._rpc = RPC(freqtrade) + config = freqtrade.config # Enable telegram - if freqtrade.config.get('telegram', {}).get('enabled', False): + if config.get('telegram', {}).get('enabled', False): logger.info('Enabling rpc.telegram ...') from freqtrade.rpc.telegram import Telegram - self.registered_modules.append(Telegram(freqtrade)) + self.registered_modules.append(Telegram(self._rpc, config)) # Enable Webhook - if freqtrade.config.get('webhook', {}).get('enabled', False): + if config.get('webhook', {}).get('enabled', False): logger.info('Enabling rpc.webhook ...') from freqtrade.rpc.webhook import Webhook - self.registered_modules.append(Webhook(freqtrade)) + self.registered_modules.append(Webhook(self._rpc, config)) # Enable local rest api server for cmd line control - if freqtrade.config.get('api_server', {}).get('enabled', False): + if config.get('api_server', {}).get('enabled', False): logger.info('Enabling rpc.api_server') from freqtrade.rpc.api_server import ApiServer - self.registered_modules.append(ApiServer(freqtrade)) + self.registered_modules.append(ApiServer(self._rpc, config)) def cleanup(self) -> None: """ Stops all enabled rpc modules """ @@ -62,7 +63,7 @@ class RPCManager: except NotImplementedError: logger.error(f"Message type '{msg['type']}' not implemented by handler {mod.name}.") - def startup_messages(self, config: Dict[str, Any], pairlist) -> None: + def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None: if config['dry_run']: self.send_msg({ 'type': RPCMessageType.WARNING_NOTIFICATION, @@ -90,3 +91,9 @@ class RPCManager: 'status': f'Searching for {stake_currency} pairs to buy and sell ' f'based on {pairlist.short_desc()}' }) + if len(protections.name_list) > 0: + prots = '\n'.join([p for prot in protections.short_desc() for k, p in prot.items()]) + self.send_msg({ + 'type': RPCMessageType.STARTUP_NOTIFICATION, + 'status': f'Using Protections: \n{prots}' + }) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 31d5bbfbd..7ec67e5d0 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,18 +5,20 @@ This module manage Telegram communication """ import json import logging -from typing import Any, Callable, Dict +from datetime import timedelta +from itertools import chain +from typing import Any, Callable, Dict, List, Union import arrow from tabulate import tabulate -from telegram import ParseMode, ReplyKeyboardMarkup, Update +from telegram import KeyboardButton, ParseMode, ReplyKeyboardMarkup, Update from telegram.error import NetworkError, TelegramError from telegram.ext import CallbackContext, CommandHandler, Updater from telegram.utils.helpers import escape_markdown from freqtrade.__init__ import __version__ -from freqtrade.rpc import RPC, RPCException, RPCMessageType -from freqtrade.rpc.fiat_convert import CryptoToFiatConverter +from freqtrade.exceptions import OperationalException +from freqtrade.rpc import RPC, RPCException, RPCHandler, RPCMessageType logger = logging.getLogger(__name__) @@ -60,22 +62,60 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: return wrapper -class Telegram(RPC): +class Telegram(RPCHandler): """ This class handles all telegram communication """ - def __init__(self, freqtrade) -> None: + def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: + """ - Init the Telegram call, and init the super class RPC - :param freqtrade: Instance of a freqtrade bot + Init the Telegram call, and init the super class RPCHandler + :param rpc: instance of RPC Helper class + :param config: Configuration object :return: None """ - super().__init__(freqtrade) + super().__init__(rpc, config) - self._updater: Updater = None - self._config = freqtrade.config + self._updater: Updater + self._init_keyboard() self._init() - if self._config.get('fiat_display_currency', None): - self._fiat_converter = CryptoToFiatConverter() + + def _init_keyboard(self) -> None: + """ + Validates the keyboard configuration from telegram config + section. + """ + self._keyboard: List[List[Union[str, KeyboardButton]]] = [ + ['/daily', '/profit', '/balance'], + ['/status', '/status table', '/performance'], + ['/count', '/start', '/stop', '/help'] + ] + # do not allow commands with mandatory arguments and critical cmds + # like /forcesell and /forcebuy + # TODO: DRY! - its not good to list all valid cmds here. But otherwise + # this needs refacoring of the whole telegram module (same + # problem in _help()). + valid_keys: List[str] = ['/start', '/stop', '/status', '/status table', + '/trades', '/profit', '/performance', '/daily', + '/stats', '/count', '/locks', '/balance', + '/stopbuy', '/reload_config', '/show_config', + '/logs', '/whitelist', '/blacklist', '/edge', + '/help', '/version'] + + # custom keyboard specified in config.json + cust_keyboard = self._config['telegram'].get('keyboard', []) + if cust_keyboard: + # check for valid shortcuts + invalid_keys = [b for b in chain.from_iterable(cust_keyboard) + if b not in valid_keys] + if len(invalid_keys): + err_msg = ('config.telegram.keyboard: Invalid commands for ' + f'custom Telegram keyboard: {invalid_keys}' + f'\nvalid commands are: {valid_keys}') + raise OperationalException(err_msg) + else: + self._keyboard = cust_keyboard + logger.info('using custom keyboard from ' + f'config.json: {self._keyboard}') def _init(self) -> None: """ @@ -98,6 +138,7 @@ class Telegram(RPC): CommandHandler('trades', self._trades), CommandHandler('delete', self._delete_trade), CommandHandler('performance', self._performance), + CommandHandler('stats', self._stats), CommandHandler('daily', self._daily), CommandHandler('count', self._count), CommandHandler('locks', self._locks), @@ -142,8 +183,8 @@ class Telegram(RPC): return if msg['type'] == RPCMessageType.BUY_NOTIFICATION: - if self._fiat_converter: - msg['stake_amount_fiat'] = self._fiat_converter.convert_amount( + if self._rpc._fiat_converter: + msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount( msg['stake_amount'], msg['stake_currency'], msg['fiat_currency']) else: msg['stake_amount_fiat'] = 0 @@ -183,8 +224,8 @@ class Telegram(RPC): # Check if all sell properties are available. # This might not be the case if the message origin is triggered by /forcesell if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency']) - and self._fiat_converter): - msg['profit_fiat'] = self._fiat_converter.convert_amount( + and self._rpc._fiat_converter): + msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount( msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) message += (' `({gain}: {profit_amount:.8f} {stake_currency}' ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg) @@ -231,12 +272,12 @@ class Telegram(RPC): :return: None """ - if 'table' in context.args: + if context.args and 'table' in context.args: self._status_table(update, context) return try: - results = self._rpc_trade_status() + results = self._rpc._rpc_trade_status() messages = [] for r in results: @@ -286,8 +327,9 @@ class Telegram(RPC): :return: None """ try: - statlist, head = self._rpc_status_table(self._config['stake_currency'], - self._config.get('fiat_display_currency', '')) + statlist, head = self._rpc._rpc_status_table( + self._config['stake_currency'], self._config.get('fiat_display_currency', '')) + message = tabulate(statlist, headers=head, tablefmt='simple') self._send_msg(f"
{message}
", parse_mode=ParseMode.HTML) except RPCException as e: @@ -305,11 +347,11 @@ class Telegram(RPC): stake_cur = self._config['stake_currency'] fiat_disp_cur = self._config.get('fiat_display_currency', '') try: - timescale = int(context.args[0]) + timescale = int(context.args[0]) if context.args else 7 except (TypeError, ValueError, IndexError): timescale = 7 try: - stats = self._rpc_daily_profit( + stats = self._rpc._rpc_daily_profit( timescale, stake_cur, fiat_disp_cur @@ -343,7 +385,7 @@ class Telegram(RPC): stake_cur = self._config['stake_currency'] fiat_disp_cur = self._config.get('fiat_display_currency', '') - stats = self._rpc_trade_statistics( + stats = self._rpc._rpc_trade_statistics( stake_cur, fiat_disp_cur) profit_closed_coin = stats['profit_closed_coin'] @@ -388,12 +430,54 @@ class Telegram(RPC): f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") self._send_msg(markdown_msg) + @authorized_only + def _stats(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /stats + Show stats of recent trades + """ + stats = self._rpc._rpc_stats() + + reason_map = { + 'roi': 'ROI', + 'stop_loss': 'Stoploss', + 'trailing_stop_loss': 'Trail. Stop', + 'stoploss_on_exchange': 'Stoploss', + 'sell_signal': 'Sell Signal', + 'force_sell': 'Forcesell', + 'emergency_sell': 'Emergency Sell', + } + sell_reasons_tabulate = [ + [ + reason_map.get(reason, reason), + sum(count.values()), + count['wins'], + count['losses'] + ] for reason, count in stats['sell_reasons'].items() + ] + sell_reasons_msg = tabulate( + sell_reasons_tabulate, + headers=['Sell Reason', 'Sells', 'Wins', 'Losses'] + ) + durations = stats['durations'] + duration_msg = tabulate([ + ['Wins', str(timedelta(seconds=durations['wins'])) + if durations['wins'] != 'N/A' else 'N/A'], + ['Losses', str(timedelta(seconds=durations['losses'])) + if durations['losses'] != 'N/A' else 'N/A'] + ], + headers=['', 'Avg. Duration'] + ) + msg = (f"""```\n{sell_reasons_msg}```\n```\n{duration_msg}```""") + + self._send_msg(msg, ParseMode.MARKDOWN) + @authorized_only def _balance(self, update: Update, context: CallbackContext) -> None: """ Handler for /balance """ try: - result = self._rpc_balance(self._config['stake_currency'], - self._config.get('fiat_display_currency', '')) + result = self._rpc._rpc_balance(self._config['stake_currency'], + self._config.get('fiat_display_currency', '')) output = '' if self._config['dry_run']: @@ -436,7 +520,7 @@ class Telegram(RPC): :param update: message update :return: None """ - msg = self._rpc_start() + msg = self._rpc._rpc_start() self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only @@ -448,7 +532,7 @@ class Telegram(RPC): :param update: message update :return: None """ - msg = self._rpc_stop() + msg = self._rpc._rpc_stop() self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only @@ -460,7 +544,7 @@ class Telegram(RPC): :param update: message update :return: None """ - msg = self._rpc_reload_config() + msg = self._rpc._rpc_reload_config() self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only @@ -472,7 +556,7 @@ class Telegram(RPC): :param update: message update :return: None """ - msg = self._rpc_stopbuy() + msg = self._rpc._rpc_stopbuy() self._send_msg('Status: `{status}`'.format(**msg)) @authorized_only @@ -485,9 +569,12 @@ class Telegram(RPC): :return: None """ - trade_id = context.args[0] if len(context.args) > 0 else None + trade_id = context.args[0] if context.args and len(context.args) > 0 else None + if not trade_id: + self._send_msg("You must specify a trade-id or 'all'.") + return try: - msg = self._rpc_forcesell(trade_id) + msg = self._rpc._rpc_forcesell(trade_id) self._send_msg('Forcesell Result: `{result}`'.format(**msg)) except RPCException as e: @@ -502,13 +589,13 @@ class Telegram(RPC): :param update: message update :return: None """ - - pair = context.args[0] - price = float(context.args[1]) if len(context.args) > 1 else None - try: - self._rpc_forcebuy(pair, price) - except RPCException as e: - self._send_msg(str(e)) + if context.args: + pair = context.args[0] + price = float(context.args[1]) if len(context.args) > 1 else None + try: + self._rpc._rpc_forcebuy(pair, price) + except RPCException as e: + self._send_msg(str(e)) @authorized_only def _trades(self, update: Update, context: CallbackContext) -> None: @@ -521,11 +608,11 @@ class Telegram(RPC): """ stake_cur = self._config['stake_currency'] try: - nrecent = int(context.args[0]) + nrecent = int(context.args[0]) if context.args else 10 except (TypeError, ValueError, IndexError): nrecent = 10 try: - trades = self._rpc_trade_history( + trades = self._rpc._rpc_trade_history( nrecent ) trades_tab = tabulate( @@ -554,10 +641,11 @@ class Telegram(RPC): :param update: message update :return: None """ - - trade_id = context.args[0] if len(context.args) > 0 else None try: - msg = self._rpc_delete(trade_id) + if not context.args or len(context.args) == 0: + raise RPCException("Trade-id not set.") + trade_id = int(context.args[0]) + msg = self._rpc._rpc_delete(trade_id) self._send_msg(( '`{result_msg}`\n' 'Please make sure to take care of this asset on the exchange manually.' @@ -576,7 +664,7 @@ class Telegram(RPC): :return: None """ try: - trades = self._rpc_performance() + trades = self._rpc._rpc_performance() stats = '\n'.join('{index}.\t{pair}\t{profit:.2f}% ({count})'.format( index=i + 1, pair=trade['pair'], @@ -598,7 +686,7 @@ class Telegram(RPC): :return: None """ try: - counts = self._rpc_count() + counts = self._rpc._rpc_count() message = tabulate({k: [v] for k, v in counts.items()}, headers=['current', 'max', 'total stake'], tablefmt='simple') @@ -615,7 +703,7 @@ class Telegram(RPC): Returns the currently active locks """ try: - locks = self._rpc_locks() + locks = self._rpc._rpc_locks() message = tabulate([[ lock['pair'], lock['lock_end_time'], @@ -635,7 +723,7 @@ class Telegram(RPC): Shows the currently active whitelist """ try: - whitelist = self._rpc_whitelist() + whitelist = self._rpc._rpc_whitelist() message = f"Using whitelist `{whitelist['method']}` with {whitelist['length']} pairs\n" message += f"`{', '.join(whitelist['whitelist'])}`" @@ -653,7 +741,7 @@ class Telegram(RPC): """ try: - blacklist = self._rpc_blacklist(context.args) + blacklist = self._rpc._rpc_blacklist(context.args) errmsgs = [] for pair, error in blacklist['errors'].items(): errmsgs.append(f"Error adding `{pair}` to blacklist: `{error['error_msg']}`") @@ -676,10 +764,10 @@ class Telegram(RPC): """ try: try: - limit = int(context.args[0]) + limit = int(context.args[0]) if context.args else 10 except (TypeError, ValueError, IndexError): limit = 10 - logs = self._rpc_get_logs(limit)['logs'] + logs = RPC._rpc_get_logs(limit)['logs'] msgs = '' msg_template = "*{}* {}: {} \\- `{}`" for logrec in logs: @@ -707,7 +795,7 @@ class Telegram(RPC): Shows information related to Edge """ try: - edge_pairs = self._rpc_edge() + edge_pairs = self._rpc._rpc_edge() edge_pairs_tab = tabulate(edge_pairs, headers='keys', tablefmt='simple') message = f'Edge only validated following pairs:\n
{edge_pairs_tab}
' self._send_msg(message, parse_mode=ParseMode.HTML) @@ -739,6 +827,8 @@ class Telegram(RPC): "*/delete :* `Instantly delete the given trade in the database`\n" "*/performance:* `Show performance of each finished trade grouped by pair`\n" "*/daily :* `Shows profit or loss per day, over the last n days`\n" + "*/stats:* `Shows Wins / losses by Sell reason as well as " + "Avg. holding durationsfor buys and sells.`\n" "*/count:* `Show number of active trades compared to allowed number of trades`\n" "*/locks:* `Show currently locked pairs`\n" "*/balance:* `Show account balance per currency`\n" @@ -775,7 +865,7 @@ class Telegram(RPC): :param update: message update :return: None """ - val = RPC._rpc_show_config(self._freqtrade.config, self._freqtrade.state) + val = RPC._rpc_show_config(self._config, self._rpc._freqtrade.state) if val['trailing_stop']: sl_info = ( @@ -802,7 +892,7 @@ class Telegram(RPC): f"*Current state:* `{val['state']}`" ) - def _send_msg(self, msg: str, parse_mode: ParseMode = ParseMode.MARKDOWN, + def _send_msg(self, msg: str, parse_mode: str = ParseMode.MARKDOWN, disable_notification: bool = False) -> None: """ Send given markdown message @@ -811,13 +901,7 @@ class Telegram(RPC): :param parse_mode: telegram parse mode :return: None """ - - keyboard = [['/daily', '/profit', '/balance'], - ['/status', '/status table', '/performance'], - ['/count', '/start', '/stop', '/help']] - - reply_markup = ReplyKeyboardMarkup(keyboard) - + reply_markup = ReplyKeyboardMarkup(self._keyboard) try: try: self._updater.bot.send_message( diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 21413f165..5796201b5 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -6,7 +6,7 @@ from typing import Any, Dict from requests import RequestException, post -from freqtrade.rpc import RPC, RPCMessageType +from freqtrade.rpc import RPC, RPCHandler, RPCMessageType logger = logging.getLogger(__name__) @@ -14,18 +14,18 @@ logger = logging.getLogger(__name__) logger.debug('Included module rpc.webhook ...') -class Webhook(RPC): +class Webhook(RPCHandler): """ This class handles all webhook communication """ - def __init__(self, freqtrade) -> None: + def __init__(self, rpc: RPC, config: Dict[str, Any]) -> None: """ - Init the Webhook class, and init the super class RPC - :param freqtrade: Instance of a freqtrade bot + Init the Webhook class, and init the super class RPCHandler + :param rpc: instance of RPC Helper class + :param config: Configuration object :return: None """ - super().__init__(freqtrade) + super().__init__(rpc, config) - self._config = freqtrade.config self._url = self._config['webhook']['url'] def cleanup(self) -> None: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 1c6aa535d..027c5d36e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -312,7 +312,7 @@ class IStrategy(ABC): if not candle_date: # Simple call ... - return PairLocks.is_pair_locked(pair, candle_date) + return PairLocks.is_pair_locked(pair) else: lock_time = timeframe_to_next_date(self.timeframe, candle_date) return PairLocks.is_pair_locked(pair, lock_time) @@ -476,40 +476,44 @@ class IStrategy(ABC): current_time=date, current_profit=current_profit, force_stoploss=force_stoploss, high=high) - if stoplossflag.sell_flag: - logger.debug(f"{trade.pair} - Stoploss hit. sell_flag=True, " - f"sell_type={stoplossflag.sell_type}") - return stoplossflag - # Set current rate to high for backtesting sell current_rate = high or rate current_profit = trade.calc_profit_ratio(current_rate) config_ask_strategy = self.config.get('ask_strategy', {}) - if buy and config_ask_strategy.get('ignore_roi_if_buy_signal', False): - # This one is noisy, commented out - # logger.debug(f"{trade.pair} - Buy signal still active. sell_flag=False") - return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) + # if buy signal and ignore_roi is set, we don't need to evaluate min_roi. + roi_reached = (not (buy and config_ask_strategy.get('ignore_roi_if_buy_signal', False)) + and self.min_roi_reached(trade=trade, current_profit=current_profit, + current_time=date)) - # Check if minimal roi has been reached and no longer in buy conditions (avoiding a fee) - if self.min_roi_reached(trade=trade, current_profit=current_profit, current_time=date): + if config_ask_strategy.get('sell_profit_only', False) and trade.calc_profit(rate=rate) <= 0: + # Negative profits and sell_profit_only - ignore sell signal + sell_signal = False + else: + sell_signal = sell and not buy and config_ask_strategy.get('use_sell_signal', True) + # TODO: return here if sell-signal should be favored over ROI + + # Start evaluations + # Sequence: + # ROI (if not stoploss) + # Sell-signal + # Stoploss + if roi_reached and stoplossflag.sell_type != SellType.STOP_LOSS: logger.debug(f"{trade.pair} - Required profit reached. sell_flag=True, " f"sell_type=SellType.ROI") return SellCheckTuple(sell_flag=True, sell_type=SellType.ROI) - if config_ask_strategy.get('sell_profit_only', False): - # This one is noisy, commented out - # logger.debug(f"{trade.pair} - Checking if trade is profitable...") - if trade.calc_profit(rate=rate) <= 0: - # This one is noisy, commented out - # logger.debug(f"{trade.pair} - Trade is not profitable. sell_flag=False") - return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) - - if sell and not buy and config_ask_strategy.get('use_sell_signal', True): + if sell_signal: logger.debug(f"{trade.pair} - Sell signal received. sell_flag=True, " f"sell_type=SellType.SELL_SIGNAL") return SellCheckTuple(sell_flag=True, sell_type=SellType.SELL_SIGNAL) + if stoplossflag.sell_flag: + + logger.debug(f"{trade.pair} - Stoploss hit. sell_flag=True, " + f"sell_type={stoplossflag.sell_type}") + return stoplossflag + # This one is noisy, commented out... # logger.debug(f"{trade.pair} - No sell signal. sell_flag=False") return SellCheckTuple(sell_flag=False, sell_type=SellType.NONE) @@ -547,8 +551,7 @@ class IStrategy(ABC): # evaluate if the stoploss was hit if stoploss is not on exchange # in Dry-Run, this handles stoploss logic as well, as the logic will not be different to # regular stoploss handling. - if ((self.stoploss is not None) and - (trade.stop_loss >= current_rate) and + if ((trade.stop_loss >= current_rate) and (not self.order_types.get('stoploss_on_exchange') or self.config['dry_run'])): sell_type = SellType.STOP_LOSS diff --git a/mkdocs.yml b/mkdocs.yml index 2cc0c9fcb..96cfa7651 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,6 +19,7 @@ nav: - Backtesting: backtesting.md - Hyperopt: hyperopt.md - Edge Positioning: edge.md + - Plugins: plugins.md - Utility Subcommands: utils.md - FAQ: faq.md - Data Analysis: @@ -31,11 +32,13 @@ nav: - Advanced Strategy: strategy-advanced.md - Advanced Hyperopt: advanced-hyperopt.md - Sandbox Testing: sandbox-testing.md + - Updating Freqtrade: updating.md - Deprecated Features: deprecated.md - Contributors Guide: developer.md theme: name: material logo: 'images/logo.png' + favicon: 'images/logo.png' custom_dir: 'docs' palette: primary: 'blue grey' diff --git a/requirements-dev.txt b/requirements-dev.txt index e681274c8..a2da87430 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,12 +6,12 @@ coveralls==2.2.0 flake8==3.8.4 flake8-type-annotations==0.1.0 -flake8-tidy-imports==4.1.0 +flake8-tidy-imports==4.2.1 mypy==0.790 -pytest==6.1.2 +pytest==6.2.1 pytest-asyncio==0.14.0 pytest-cov==2.10.1 -pytest-mock==3.3.1 +pytest-mock==3.4.0 pytest-random-order==1.0.4 isort==5.6.4 diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 7e480b8c9..c51062bf7 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -6,5 +6,5 @@ scipy==1.5.4 scikit-learn==0.23.2 scikit-optimize==0.8.1 filelock==3.0.12 -joblib==0.17.0 +joblib==1.0.0 progressbar2==3.53.1 diff --git a/requirements-plot.txt b/requirements-plot.txt index bd40bc0b5..3e31a24ae 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==4.12.0 +plotly==4.14.1 diff --git a/requirements.txt b/requirements.txt index 7490688d4..594c22b74 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ numpy==1.19.4 -pandas==1.1.4 +pandas==1.1.5 -ccxt==1.38.13 +ccxt==1.39.79 aiohttp==3.7.3 -SQLAlchemy==1.3.20 -python-telegram-bot==13.0 +SQLAlchemy==1.3.22 +python-telegram-bot==13.1 arrow==0.17.0 -cachetools==4.1.1 -requests==2.25.0 +cachetools==4.2.0 +requests==2.25.1 urllib3==1.26.2 wrapt==1.12.1 jsonschema==3.2.0 @@ -16,13 +16,13 @@ tabulate==0.8.7 pycoingecko==1.4.0 jinja2==2.11.2 tables==3.6.1 -blosc==1.9.2 +blosc==1.10.1 # find first, C search in arrays py_find_1st==1.1.4 # Load ticker files 30% faster -python-rapidjson==0.9.4 +python-rapidjson==1.0 # Notify systemd sdnotify==0.3.2 @@ -35,5 +35,5 @@ flask-cors==3.0.9 # Support for colorized terminal output colorama==0.4.4 # Building config files interactively -questionary==1.8.1 +questionary==1.9.0 prompt-toolkit==3.0.8 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index 268e81397..2232b8421 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -139,6 +139,13 @@ class FtRestClient(): """ return self._get("profit") + def stats(self): + """Return the stats report (durations, sell-reasons). + + :return: json object + """ + return self._get("stats") + def performance(self): """Return the performance of the different coins. diff --git a/setup.py b/setup.py index b47427709..030980c96 100644 --- a/setup.py +++ b/setup.py @@ -3,9 +3,9 @@ from sys import version_info from setuptools import setup -if version_info.major == 3 and version_info.minor < 6 or \ +if version_info.major == 3 and version_info.minor < 7 or \ version_info.major < 3: - print('Your Python interpreter must be 3.6 or greater!') + print('Your Python interpreter must be 3.7 or greater!') exit(1) from pathlib import Path # noqa: E402 @@ -109,7 +109,6 @@ setup(name='freqtrade', 'Environment :: Console', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Operating System :: MacOS', diff --git a/setup.sh b/setup.sh index 049a6a77e..b270146c1 100755 --- a/setup.sh +++ b/setup.sh @@ -25,6 +25,14 @@ function check_installed_python() { return fi + which python3.9 + if [ $? -eq 0 ]; then + echo "using Python 3.9" + PYTHON=python3.9 + check_installed_pip + return + fi + which python3.7 if [ $? -eq 0 ]; then echo "using Python 3.7" @@ -33,16 +41,9 @@ function check_installed_python() { return fi - which python3.6 - if [ $? -eq 0 ]; then - echo "using Python 3.6" - PYTHON=python3.6 - check_installed_pip - return - fi if [ -z ${PYTHON} ]; then - echo "No usable python found. Please make sure to have python3.6 or python3.7 installed" + echo "No usable python found. Please make sure to have python3.7 or newer installed" exit 1 fi } @@ -56,18 +57,45 @@ function updateenv() { exit 1 fi source .env/bin/activate + SYS_ARCH=$(uname -m) echo "pip install in-progress. Please wait..." ${PYTHON} -m pip install --upgrade pip read -p "Do you want to install dependencies for dev [y/N]? " if [[ $REPLY =~ ^[Yy]$ ]] then - ${PYTHON} -m pip install --upgrade -r requirements-dev.txt + REQUIREMENTS=requirements-dev.txt else - ${PYTHON} -m pip install --upgrade -r requirements.txt - echo "Dev dependencies ignored." + REQUIREMENTS=requirements.txt + fi + REQUIREMENTS_HYPEROPT="" + REQUIREMENTS_PLOT="" + read -p "Do you want to install plotting dependencies (plotly) [y/N]? " + if [[ $REPLY =~ ^[Yy]$ ]] + then + REQUIREMENTS_PLOT="-r requirements-plot.txt" + fi + if [ "${SYS_ARCH}" == "armv7l" ]; then + echo "Detected Raspberry, installing cython, skipping hyperopt installation." + ${PYTHON} -m pip install --upgrade cython + else + # Is not Raspberry + read -p "Do you want to install hyperopt dependencies [y/N]? " + if [[ $REPLY =~ ^[Yy]$ ]] + then + REQUIREMENTS_HYPEROPT="-r requirements-hyperopt.txt" + fi fi + ${PYTHON} -m pip install --upgrade -r ${REQUIREMENTS} ${REQUIREMENTS_HYPEROPT} ${REQUIREMENTS_PLOT} + if [ $? -ne 0 ]; then + echo "Failed installing dependencies" + exit 1 + fi ${PYTHON} -m pip install -e . + if [ $? -ne 0 ]; then + echo "Failed installing Freqtrade" + exit 1 + fi echo "pip install completed" echo } @@ -134,11 +162,11 @@ function reset() { git fetch -a - if [ "1" == $(git branch -vv |grep -c "* develop") ] + if [ "1" == $(git branch -vv | grep -c "* develop") ] then echo "- Hard resetting of 'develop' branch." git reset --hard origin/develop - elif [ "1" == $(git branch -vv |grep -c "* stable") ] + elif [ "1" == $(git branch -vv | grep -c "* stable") ] then echo "- Hard resetting of 'stable' branch." git reset --hard origin/stable @@ -149,7 +177,7 @@ function reset() { fi if [ -d ".env" ]; then - echo "- Delete your previous virtual env" + echo "- Deleting your previous virtual env" rm -rf .env fi echo @@ -253,7 +281,7 @@ function install() { echo "Run the bot !" echo "-------------------------" echo "You can now use the bot by executing 'source .env/bin/activate; freqtrade '." - echo "You can see the list of available bot subcommands by executing 'source .env/bin/activate; freqtrade --help'." + echo "You can see the list of available bot sub-commands by executing 'source .env/bin/activate; freqtrade --help'." echo "You verify that freqtrade is installed successfully by running 'source .env/bin/activate; freqtrade --version'." } @@ -275,7 +303,7 @@ function help() { echo " -p,--plot Install dependencies for Plotting scripts." } -# Verify if 3.6 or 3.7 is installed +# Verify if 3.7 or 3.8 is installed check_installed_python case $* in diff --git a/tests/conftest.py b/tests/conftest.py index 079a521ed..9eda0e973 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,6 +33,19 @@ logging.getLogger('').setLevel(logging.INFO) np.seterr(all='raise') +def pytest_addoption(parser): + parser.addoption('--longrun', action='store_true', dest="longrun", + default=False, help="Enable long-run tests (ccxt compat)") + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "longrun: mark test that is running slowly and should not be run regularily" + ) + if not config.option.longrun: + setattr(config.option, 'markexpr', 'not longrun') + + def log_has(line, logs): # caplog mocker returns log as a tuple: ('freqtrade.something', logging.WARNING, 'foobar') # and we want to match line against foobar in the tuple @@ -224,6 +237,10 @@ def init_persistence(default_conf): @pytest.fixture(scope="function") def default_conf(testdatadir): + return get_default_conf(testdatadir) + + +def get_default_conf(testdatadir): """ Returns validated configuration suitable for most tests """ configuration = { "max_open_trades": 1, @@ -1084,7 +1101,7 @@ def ohlcv_history_list(): @pytest.fixture def ohlcv_history(ohlcv_history_list): return ohlcv_to_dataframe(ohlcv_history_list, "5m", pair="UNITTEST/BTC", - fill_missing=True) + fill_missing=True, drop_incomplete=False) @pytest.fixture @@ -1588,16 +1605,7 @@ def fetch_trades_result(): @pytest.fixture(scope="function") def trades_for_order2(): - return [{'info': {'id': 34567, - 'orderId': 123456, - 'price': '0.24544100', - 'qty': '8.00000000', - 'commission': '0.00800000', - 'commissionAsset': 'LTC', - 'time': 1521663363189, - 'isBuyer': True, - 'isMaker': False, - 'isBestMatch': True}, + return [{'info': {}, 'timestamp': 1521663363189, 'datetime': '2018-03-21T20:16:03.189Z', 'symbol': 'LTC/ETH', @@ -1609,16 +1617,7 @@ def trades_for_order2(): 'cost': 1.963528, 'amount': 4.0, 'fee': {'cost': 0.004, 'currency': 'LTC'}}, - {'info': {'id': 34567, - 'orderId': 123456, - 'price': '0.24544100', - 'qty': '8.00000000', - 'commission': '0.00800000', - 'commissionAsset': 'LTC', - 'time': 1521663363189, - 'isBuyer': True, - 'isMaker': False, - 'isBestMatch': True}, + {'info': {}, 'timestamp': 1521663363189, 'datetime': '2018-03-21T20:16:03.189Z', 'symbol': 'LTC/ETH', @@ -1632,6 +1631,14 @@ def trades_for_order2(): 'fee': {'cost': 0.004, 'currency': 'LTC'}}] +@pytest.fixture(scope="function") +def trades_for_order3(trades_for_order2): + # Different fee currencies for each trade + trades_for_order = deepcopy(trades_for_order2) + trades_for_order[0]['fee'] = {'cost': 0.02, 'currency': 'BNB'} + return trades_for_order + + @pytest.fixture def buy_order_fee(): return { diff --git a/tests/conftest_trades.py b/tests/conftest_trades.py index 78388f022..e84722041 100644 --- a/tests/conftest_trades.py +++ b/tests/conftest_trades.py @@ -1,3 +1,5 @@ +from datetime import datetime, timedelta, timezone + from freqtrade.persistence.models import Order, Trade @@ -82,6 +84,9 @@ def mock_trade_2(fee): is_open=False, open_order_id='dry_run_sell_12345', strategy='DefaultStrategy', + sell_reason='sell_signal', + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc), ) o = Order.parse_from_ccxt_object(mock_order_2(), 'ETC/BTC', 'buy') trade.orders.append(o) @@ -134,6 +139,9 @@ def mock_trade_3(fee): close_profit=0.01, exchange='bittrex', is_open=False, + sell_reason='roi', + open_date=datetime.now(tz=timezone.utc) - timedelta(minutes=20), + close_date=datetime.now(tz=timezone.utc), ) o = Order.parse_from_ccxt_object(mock_order_3(), 'XRP/BTC', 'buy') trade.orders.append(o) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index a3c57a77b..ee2e551b6 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -6,7 +6,7 @@ from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import ExchangeError, OperationalException -from freqtrade.pairlist.pairlistmanager import PairListManager +from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.state import RunMode from tests.conftest import get_patched_exchange diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py new file mode 100644 index 000000000..0c8b7bdcf --- /dev/null +++ b/tests/exchange/test_ccxt_compat.py @@ -0,0 +1,134 @@ +""" +Tests in this file do NOT mock network calls, so they are expected to be fluky at times. + +However, these tests should give a good idea to determine if a new exchange is +suitable to run with freqtrade. +""" + +from pathlib import Path + +import pytest + +from freqtrade.resolvers.exchange_resolver import ExchangeResolver +from tests.conftest import get_default_conf + + +# Exchanges that should be tested +EXCHANGES = { + 'bittrex': { + 'pair': 'BTC/USDT', + 'hasQuoteVolume': False, + 'timeframe': '5m', + }, + 'binance': { + 'pair': 'BTC/USDT', + 'hasQuoteVolume': True, + 'timeframe': '5m', + }, + 'kraken': { + 'pair': 'BTC/USDT', + 'hasQuoteVolume': True, + 'timeframe': '5m', + }, + 'ftx': { + 'pair': 'BTC/USDT', + 'hasQuoteVolume': True, + 'timeframe': '5m', + } +} + + +@pytest.fixture(scope="class") +def exchange_conf(): + config = get_default_conf((Path(__file__).parent / "testdata").resolve()) + config['exchange']['pair_whitelist'] = [] + return config + + +@pytest.fixture(params=EXCHANGES, scope="class") +def exchange(request, exchange_conf): + exchange_conf['exchange']['name'] = request.param + exchange = ExchangeResolver.load_exchange(request.param, exchange_conf, validate=True) + + yield exchange, request.param + + +@pytest.mark.longrun +class TestCCXTExchange(): + + def test_load_markets(self, exchange): + exchange, exchangename = exchange + pair = EXCHANGES[exchangename]['pair'] + markets = exchange.markets + assert pair in markets + assert isinstance(markets[pair], dict) + + def test_ccxt_fetch_tickers(self, exchange): + exchange, exchangename = exchange + pair = EXCHANGES[exchangename]['pair'] + + tickers = exchange.get_tickers() + assert pair in tickers + assert 'ask' in tickers[pair] + assert tickers[pair]['ask'] is not None + assert 'bid' in tickers[pair] + assert tickers[pair]['bid'] is not None + assert 'quoteVolume' in tickers[pair] + if EXCHANGES[exchangename].get('hasQuoteVolume'): + assert tickers[pair]['quoteVolume'] is not None + + def test_ccxt_fetch_ticker(self, exchange): + exchange, exchangename = exchange + pair = EXCHANGES[exchangename]['pair'] + + ticker = exchange.fetch_ticker(pair) + assert 'ask' in ticker + assert ticker['ask'] is not None + assert 'bid' in ticker + assert ticker['bid'] is not None + assert 'quoteVolume' in ticker + if EXCHANGES[exchangename].get('hasQuoteVolume'): + assert ticker['quoteVolume'] is not None + + def test_ccxt_fetch_l2_orderbook(self, exchange): + exchange, exchangename = exchange + pair = EXCHANGES[exchangename]['pair'] + l2 = exchange.fetch_l2_order_book(pair) + assert 'asks' in l2 + assert 'bids' in l2 + l2_limit_range = exchange._ft_has['l2_limit_range'] + for val in [1, 2, 5, 25, 100]: + l2 = exchange.fetch_l2_order_book(pair, val) + if not l2_limit_range or val in l2_limit_range: + assert len(l2['asks']) == val + assert len(l2['bids']) == val + else: + next_limit = exchange.get_next_limit_in_list(val, l2_limit_range) + if next_limit > 200: + # Large orderbook sizes can be a problem for some exchanges (bitrex ...) + assert len(l2['asks']) > 200 + assert len(l2['asks']) > 200 + else: + assert len(l2['asks']) == next_limit + assert len(l2['asks']) == next_limit + + def test_fetch_ohlcv(self, exchange): + exchange, exchangename = exchange + pair = EXCHANGES[exchangename]['pair'] + timeframe = EXCHANGES[exchangename]['timeframe'] + pair_tf = (pair, timeframe) + ohlcv = exchange.refresh_latest_ohlcv([pair_tf]) + assert isinstance(ohlcv, dict) + assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf)) + assert len(exchange.klines(pair_tf)) > 200 + + # TODO: tests fetch_trades (?) + + def test_ccxt_get_fee(self, exchange): + exchange, exchangename = exchange + pair = EXCHANGES[exchangename]['pair'] + + assert 0 < exchange.get_fee(pair, 'limit', 'buy') < 1 + assert 0 < exchange.get_fee(pair, 'limit', 'sell') < 1 + assert 0 < exchange.get_fee(pair, 'market', 'buy') < 1 + assert 0 < exchange.get_fee(pair, 'market', 'sell') < 1 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 42681b367..a42ff52e4 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1385,6 +1385,12 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')] # empty dicts assert not exchange._klines + exchange.refresh_latest_ohlcv(pairs, cache=False) + # No caching + assert not exchange._klines + assert exchange._api_async.fetch_ohlcv.call_count == 2 + exchange._api_async.fetch_ohlcv.reset_mock() + exchange.refresh_latest_ohlcv(pairs) assert log_has(f'Refreshing candle (OHLCV) data for {len(pairs)} pairs', caplog) @@ -1499,11 +1505,9 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): assert exchange._klines assert exchange._api_async.fetch_ohlcv.call_count == 2 - assert type(res) is list - assert len(res) == 2 + assert type(res) is dict + assert len(res) == 1 # Test that each is in list at least once as order is not guaranteed - assert type(res[0]) is tuple or type(res[1]) is tuple - assert type(res[0]) is TypeError or type(res[1]) is TypeError assert log_has("Error loading ETH/BTC. Result was [[]].", caplog) assert log_has("Async code raised an exception: TypeError", caplog) @@ -2149,7 +2153,7 @@ def test_get_fee(default_conf, mocker, exchange_name): def test_stoploss_order_unsupported_exchange(default_conf, mocker): - exchange = get_patched_exchange(mocker, default_conf, 'bittrex') + exchange = get_patched_exchange(mocker, default_conf, id='bittrex') with pytest.raises(OperationalException, match=r"stoploss is not implemented .*"): exchange.stoploss(pair='ETH/BTC', amount=1, stop_price=220, order_types={}) diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index a5de64fe4..720ed8c13 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -328,6 +328,118 @@ tc20 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] ) +# Test 21: trailing_stop ROI collision. +# Roi should trigger before Trailing stop - otherwise Trailing stop profits can be > ROI +# which cannot happen in reality +# stop-loss: 10%, ROI: 4%, Trailing stop adjusted at the sell candle +tc21 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 4650, 5100, 6172, 0, 0], + [3, 4850, 5050, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.04}, profit_perc=0.04, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)] +) + +# Test 22: trailing_stop Raises in candle 2 - but ROI applies at the same time. +# applying a positive trailing stop of 3% - ROI should apply before trailing stop. +# stop-loss: 10%, ROI: 4%, stoploss adjusted candle 2 +tc22 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 5100, 5100, 6172, 0, 0], + [3, 4850, 5050, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.04}, profit_perc=0.04, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=2)] +) + +# Test 23: trailing_stop Raises in candle 2 (does not trigger) +# applying a positive trailing stop of 3% since stop_positive_offset is reached. +# ROI is changed after this to 4%, dropping ROI below trailing_stop_positive, causing a sell +# in the candle after the raised stoploss candle with ROI reason. +# Stoploss would trigger in this candle too, but it's no longer relevant. +# stop-loss: 10%, ROI: 4%, stoploss adjusted candle 2, ROI adjusted in candle 3 (causing the sell) +tc23 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 4950, 5100, 6172, 0, 0], + [2, 5100, 5251, 5100, 5100, 6172, 0, 0], + [3, 4850, 5251, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.1, "119": 0.03}, profit_perc=0.03, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.05, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] +) + +# Test 24: Sell with signal sell in candle 3 (stoploss also triggers on this candle) +# Stoploss at 1%. +# Stoploss wins over Sell-signal (because sell-signal is acted on in the next candle) +tc24 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4600, 6172, 0, 0], + [3, 5010, 5000, 4855, 5010, 6172, 0, 1], # Triggers stoploss + sellsignal + [4, 5010, 4987, 4977, 4995, 6172, 0, 0], + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], + stop_loss=-0.01, roi={"0": 1}, profit_perc=-0.01, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.STOP_LOSS, open_tick=1, close_tick=3)] +) + +# Test 25: Sell with signal sell in candle 3 (stoploss also triggers on this candle) +# Stoploss at 1%. +# Sell-signal wins over stoploss +tc25 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4600, 6172, 0, 0], + [3, 5010, 5000, 4986, 5010, 6172, 0, 1], + [4, 5010, 4987, 4855, 4995, 6172, 0, 0], # Triggers stoploss + sellsignal acted on + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], + stop_loss=-0.01, roi={"0": 1}, profit_perc=0.002, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.SELL_SIGNAL, open_tick=1, close_tick=4)] +) + +# Test 26: Sell with signal sell in candle 3 (ROI at signal candle) +# Stoploss at 10% (irrelevant), ROI at 5% (will trigger) +# Sell-signal wins over stoploss +tc26 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4600, 6172, 0, 0], + [3, 5010, 5251, 4986, 5010, 6172, 0, 1], # Triggers ROI, sell-signal + [4, 5010, 4987, 4855, 4995, 6172, 0, 0], + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.05}, profit_perc=0.05, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=3)] +) + +# Test 27: Sell with signal sell in candle 3 (ROI at signal candle) +# Stoploss at 10% (irrelevant), ROI at 5% (will trigger) - Wins over Sell-signal +# TODO: figure out if sell-signal should win over ROI +# Sell-signal wins over stoploss +tc27 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5025, 4975, 4987, 6172, 1, 0], + [1, 5000, 5025, 4975, 4987, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4987, 5012, 4986, 4600, 6172, 0, 0], + [3, 5010, 5012, 4986, 5010, 6172, 0, 1], # sell-signal + [4, 5010, 5251, 4855, 4995, 6172, 0, 0], # Triggers ROI, sell-signal acted on + [5, 4995, 4995, 4995, 4950, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.05}, profit_perc=0.05, use_sell_signal=True, + trades=[BTrade(sell_reason=SellType.ROI, open_tick=1, close_tick=4)] +) TESTS = [ tc0, @@ -351,6 +463,13 @@ TESTS = [ tc18, tc19, tc20, + tc21, + tc22, + tc23, + tc24, + tc25, + tc26, + tc27, ] diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 45cbea68e..376390664 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -79,7 +79,7 @@ def load_data_test(what, testdatadir): fill_missing=True)} -def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: +def simple_backtest(config, contour, mocker, testdatadir) -> None: patch_exchange(mocker) config['timeframe'] = '1m' backtesting = Backtesting(config) @@ -95,9 +95,10 @@ def simple_backtest(config, contour, num_results, mocker, testdatadir) -> None: end_date=max_date, max_open_trades=1, position_stacking=False, + enable_protections=config.get('enable_protections', False), ) # results :: - assert len(results) == num_results + return results # FIX: fixturize this? @@ -340,7 +341,7 @@ def test_backtesting_start(default_conf, mocker, testdatadir, caplog) -> None: mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats') mocker.patch('freqtrade.optimize.backtesting.show_backtest_results') - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) default_conf['timeframe'] = '1m' @@ -371,7 +372,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> mocker.patch('freqtrade.data.history.get_timerange', get_timerange) patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) default_conf['timeframe'] = "1m" @@ -391,7 +392,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> mocker.patch('freqtrade.data.history.get_timerange', get_timerange) patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=[])) default_conf['timeframe'] = "1m" @@ -414,9 +415,9 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti mocker.patch('freqtrade.data.history.get_timerange', get_timerange) patch_exchange(mocker) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['XRP/BTC'])) - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.refresh_pairlist') + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.refresh_pairlist') default_conf['ticker_interval'] = "1m" default_conf['datadir'] = testdatadir @@ -429,6 +430,11 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti with pytest.raises(OperationalException, match='VolumePairList not allowed for backtesting.'): Backtesting(default_conf) + default_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PerformanceFilter"}] + with pytest.raises(OperationalException, + match='PerformanceFilter not allowed for backtesting.'): + Backtesting(default_conf) + default_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}, ] Backtesting(default_conf) @@ -531,13 +537,52 @@ def test_processed(default_conf, mocker, testdatadir) -> None: assert col in cols -def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir) -> None: - # TODO: Evaluate usefullness of this, the patterns and buy-signls are unrealistic - mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) - tests = [['raise', 19], ['lower', 0], ['sine', 35]] +def test_backtest_pricecontours_protections(default_conf, fee, mocker, testdatadir) -> None: + # While this test IS a copy of test_backtest_pricecontours, it's needed to ensure + # results do not carry-over to the next run, which is not given by using parametrize. + default_conf['protections'] = [ + { + "method": "CooldownPeriod", + "stop_duration": 3, + }] + default_conf['enable_protections'] = True + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + tests = [ + ['sine', 9], + ['raise', 10], + ['lower', 0], + ['sine', 9], + ['raise', 10], + ] + # While buy-signals are unrealistic, running backtesting + # over and over again should not cause different results for [contour, numres] in tests: - simple_backtest(default_conf, contour, numres, mocker, testdatadir) + assert len(simple_backtest(default_conf, contour, mocker, testdatadir)) == numres + + +@pytest.mark.parametrize('protections,contour,expected', [ + (None, 'sine', 35), + (None, 'raise', 19), + (None, 'lower', 0), + (None, 'sine', 35), + (None, 'raise', 19), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'sine', 9), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'raise', 10), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'lower', 0), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'sine', 9), + ([{"method": "CooldownPeriod", "stop_duration": 3}], 'raise', 10), +]) +def test_backtest_pricecontours(default_conf, fee, mocker, testdatadir, + protections, contour, expected) -> None: + if protections: + default_conf['protections'] = protections + default_conf['enable_protections'] = True + + mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) + # While buy-signals are unrealistic, running backtesting + # over and over again should not cause different results + assert len(simple_backtest(default_conf, contour, mocker, testdatadir)) == expected def test_backtest_clash_buy_sell(mocker, default_conf, testdatadir): @@ -655,7 +700,7 @@ def test_backtest_start_timerange(default_conf, mocker, caplog, testdatadir): mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest') mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats') mocker.patch('freqtrade.optimize.backtesting.show_backtest_results') - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) patched_configuration_load_config_file(mocker, default_conf) @@ -695,7 +740,7 @@ def test_backtest_start_multi_strat(default_conf, mocker, caplog, testdatadir): patch_exchange(mocker) backtestmock = MagicMock(return_value=pd.DataFrame(columns=BT_DATA_COLUMNS + ['profit_abs'])) - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) text_table_mock = MagicMock() @@ -792,7 +837,7 @@ def test_backtest_start_multi_strat_nomock(default_conf, mocker, caplog, testdat 'sell_reason': [SellType.ROI, SellType.ROI, SellType.STOP_LOSS] }), ]) - mocker.patch('freqtrade.pairlist.pairlistmanager.PairListManager.whitelist', + mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist', PropertyMock(return_value=['UNITTEST/BTC'])) mocker.patch('freqtrade.optimize.backtesting.Backtesting.backtest', backtestmock) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index d04929164..a0e1932ff 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -76,7 +76,8 @@ def test_generate_backtest_stats(default_conf, testdatadir): "sell_reason": [SellType.ROI, SellType.STOP_LOSS, SellType.ROI, SellType.FORCE_SELL] }), - 'config': default_conf} + 'config': default_conf, + 'locks': []} } timerange = TimeRange.parse_timerange('1510688220-1510700340') min_date = Arrow.fromtimestamp(1510688220) diff --git a/tests/plugins/__init__.py b/tests/plugins/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/pairlist/test_pairlist.py b/tests/plugins/test_pairlist.py similarity index 83% rename from tests/pairlist/test_pairlist.py rename to tests/plugins/test_pairlist.py index d696e6d02..1795fc27f 100644 --- a/tests/pairlist/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -6,7 +6,7 @@ import pytest from freqtrade.constants import AVAILABLE_PAIRLISTS from freqtrade.exceptions import OperationalException -from freqtrade.pairlist.pairlistmanager import PairListManager +from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.resolvers import PairListResolver from tests.conftest import get_patched_freqtradebot, log_has, log_has_re @@ -92,7 +92,7 @@ def static_pl_conf(whitelist_conf): return whitelist_conf -def test_log_on_refresh(mocker, static_pl_conf, markets, tickers): +def test_log_cached(mocker, static_pl_conf, markets, tickers): mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), exchange_has=MagicMock(return_value=True), @@ -102,14 +102,14 @@ def test_log_on_refresh(mocker, static_pl_conf, markets, tickers): logmock = MagicMock() # Assign starting whitelist pl = freqtrade.pairlists._pairlist_handlers[0] - pl.log_on_refresh(logmock, 'Hello world') + pl.log_once('Hello world', logmock) assert logmock.call_count == 1 - pl.log_on_refresh(logmock, 'Hello world') + pl.log_once('Hello world', logmock) assert logmock.call_count == 1 assert pl._log_cache.currsize == 1 assert ('Hello world',) in pl._log_cache._Cache__data - pl.log_on_refresh(logmock, 'Hello world2') + pl.log_once('Hello world2', logmock) assert logmock.call_count == 2 assert pl._log_cache.currsize == 2 @@ -190,7 +190,7 @@ def test_refresh_pairlist_dynamic_2(mocker, shitcoinmarkets, tickers, whitelist_ ) # Remove caching of ticker data to emulate changing volume by the time of second call mocker.patch.multiple( - 'freqtrade.pairlist.pairlistmanager.PairListManager', + 'freqtrade.plugins.pairlistmanager.PairListManager', _get_cached_tickers=MagicMock(return_value=tickers_dict), ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_2) @@ -246,7 +246,7 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): {"method": "PrecisionFilter"}, {"method": "PriceFilter", "low_price_ratio": 0.03}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}, - {"method": "ShuffleFilter"}], + {"method": "ShuffleFilter"}, {"method": "PerformanceFilter"}], "ETH", []), # AgeFilter and VolumePairList (require 2 days only, all should pass age test) ([{"method": "VolumePairList", "number_assets": 5, "sort_key": "quoteVolume"}, @@ -326,6 +326,13 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): # ShuffleFilter only ([{"method": "ShuffleFilter", "seed": 42}], "BTC", 'filter_at_the_beginning'), # OperationalException expected + # PerformanceFilter after StaticPairList + ([{"method": "StaticPairList"}, + {"method": "PerformanceFilter"}], + "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), + # PerformanceFilter only + ([{"method": "PerformanceFilter"}], + "BTC", 'filter_at_the_beginning'), # OperationalException expected # SpreadFilter after StaticPairList ([{"method": "StaticPairList"}, {"method": "SpreadFilter", "max_spread_ratio": 0.005}], @@ -346,11 +353,19 @@ def test_VolumePairList_refresh_empty(mocker, markets_empty, whitelist_conf): "BTC", ['ETH/BTC', 'TKN/BTC', 'HOT/BTC']), ]) def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, tickers, - ohlcv_history_list, pairlists, base_currency, + ohlcv_history, pairlists, base_currency, whitelist_result, caplog) -> None: whitelist_conf['pairlists'] = pairlists whitelist_conf['stake_currency'] = base_currency + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history, + ('HOT/BTC', '1d'): ohlcv_history, + } + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) if whitelist_result == 'static_in_the_middle': @@ -367,9 +382,14 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), ) + # Provide for PerformanceFilter's dependency + mocker.patch.multiple('freqtrade.persistence.Trade', + get_overall_performance=MagicMock(return_value=[]) + ) + # Set whitelist_result to None if pairlist is invalid and should produce exception if whitelist_result == 'filter_at_the_beginning': with pytest.raises(OperationalException, @@ -390,7 +410,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t for pairlist in pairlists: if pairlist['method'] == 'AgeFilter' and pairlist['min_days_listed'] and \ - len(ohlcv_history_list) <= pairlist['min_days_listed']: + len(ohlcv_history) <= pairlist['min_days_listed']: assert log_has_re(r'^Removed .* from whitelist, because age .* is less than ' r'.* day.*', caplog) if pairlist['method'] == 'PrecisionFilter' and whitelist_result: @@ -413,7 +433,7 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t assert not log_has(logmsg, caplog) -def test_PrecisionFilter_error(mocker, whitelist_conf, tickers) -> None: +def test_PrecisionFilter_error(mocker, whitelist_conf) -> None: whitelist_conf['pairlists'] = [{"method": "StaticPairList"}, {"method": "PrecisionFilter"}] del whitelist_conf['stoploss'] @@ -486,7 +506,7 @@ def test__whitelist_for_active_markets(mocker, whitelist_conf, markets, pairlist @pytest.mark.parametrize("pairlist", AVAILABLE_PAIRLISTS) -def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, markets, pairlist, tickers): +def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, pairlist, tickers): whitelist_conf['pairlists'][0]['method'] = pairlist mocker.patch('freqtrade.exchange.Exchange.exchange_has', return_value=True) @@ -502,7 +522,7 @@ def test__whitelist_for_active_markets_empty(mocker, whitelist_conf, markets, pa pairlist_handler._whitelist_for_active_markets(['ETH/BTC']) -def test_volumepairlist_invalid_sortvalue(mocker, markets, whitelist_conf): +def test_volumepairlist_invalid_sortvalue(mocker, whitelist_conf): whitelist_conf['pairlists'][0].update({"sort_key": "asdf"}) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -563,8 +583,12 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick get_patched_freqtradebot(mocker, default_conf) -def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history_list): - +def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + } mocker.patch.multiple('freqtrade.exchange.Exchange', markets=PropertyMock(return_value=markets), exchange_has=MagicMock(return_value=True), @@ -572,18 +596,21 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), ) freqtrade = get_patched_freqtradebot(mocker, whitelist_conf_agefilter) - assert freqtrade.exchange.get_historic_ohlcv.call_count == 0 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 freqtrade.pairlists.refresh_pairlist() - assert freqtrade.exchange.get_historic_ohlcv.call_count > 0 + assert len(freqtrade.pairlists.whitelist) == 3 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 + # freqtrade.config['exchange']['pair_whitelist'].append('HOT/BTC') - previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count + previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count freqtrade.pairlists.refresh_pairlist() - # Should not have increased since first call. - assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count + assert len(freqtrade.pairlists.whitelist) == 3 + # Called once for XRP/BTC + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count + 1 def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): @@ -613,7 +640,7 @@ def test_rangestabilityfilter_checks(mocker, default_conf, markets, tickers): (0.01, 5), (0.05, 0), # Setting rate_of_change to 5% removes all pairs from the whitelist. ]) -def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history_list, +def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, ohlcv_history, min_rate_of_change, expected_length): default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 10}, {'method': 'RangeStabilityFilter', 'lookback_days': 2, @@ -624,22 +651,30 @@ def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, oh exchange_has=MagicMock(return_value=True), get_tickers=tickers ) + ohlcv_data = { + ('ETH/BTC', '1d'): ohlcv_history, + ('TKN/BTC', '1d'): ohlcv_history, + ('LTC/BTC', '1d'): ohlcv_history, + ('XRP/BTC', '1d'): ohlcv_history, + ('HOT/BTC', '1d'): ohlcv_history, + ('BLK/BTC', '1d'): ohlcv_history, + } mocker.patch.multiple( 'freqtrade.exchange.Exchange', - get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), + refresh_latest_ohlcv=MagicMock(return_value=ohlcv_data), ) freqtrade = get_patched_freqtradebot(mocker, default_conf) - assert freqtrade.exchange.get_historic_ohlcv.call_count == 0 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 0 freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == expected_length - assert freqtrade.exchange.get_historic_ohlcv.call_count > 0 + assert freqtrade.exchange.refresh_latest_ohlcv.call_count > 0 - previous_call_count = freqtrade.exchange.get_historic_ohlcv.call_count + previous_call_count = freqtrade.exchange.refresh_latest_ohlcv.call_count freqtrade.pairlists.refresh_pairlist() assert len(freqtrade.pairlists.whitelist) == expected_length # Should not have increased since first call. - assert freqtrade.exchange.get_historic_ohlcv.call_count == previous_call_count + assert freqtrade.exchange.refresh_latest_ohlcv.call_count == previous_call_count @pytest.mark.parametrize("pairlistconfig,desc_expected,exception_expected", [ @@ -701,7 +736,7 @@ def test_pricefilter_desc(mocker, whitelist_conf, markets, pairlistconfig, freqtrade = get_patched_freqtradebot(mocker, whitelist_conf) -def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): +def test_pairlistmanager_no_pairlist(mocker, whitelist_conf): mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) whitelist_conf['pairlists'] = [] @@ -709,3 +744,63 @@ def test_pairlistmanager_no_pairlist(mocker, markets, whitelist_conf, caplog): with pytest.raises(OperationalException, match=r"No Pairlist Handlers defined"): get_patched_freqtradebot(mocker, whitelist_conf) + + +@pytest.mark.parametrize("pairlists,pair_allowlist,overall_performance,allowlist_result", [ + # No trades yet + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], [], ['ETH/BTC', 'TKN/BTC', 'LTC/BTC']), + # Happy path: Descending order, all values filled + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC'], + [{'pair': 'TKN/BTC', 'profit': 5, 'count': 3}, {'pair': 'ETH/BTC', 'profit': 4, 'count': 2}], + ['TKN/BTC', 'ETH/BTC']), + # Performance data outside allow list ignored + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC'], + [{'pair': 'OTHER/BTC', 'profit': 5, 'count': 3}, + {'pair': 'ETH/BTC', 'profit': 4, 'count': 2}], + ['ETH/BTC', 'TKN/BTC']), + # Partial performance data missing and sorted between positive and negative profit + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], + [{'pair': 'ETH/BTC', 'profit': -5, 'count': 100}, + {'pair': 'TKN/BTC', 'profit': 4, 'count': 2}], + ['TKN/BTC', 'LTC/BTC', 'ETH/BTC']), + # Tie in performance data broken by count (ascending) + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], + [{'pair': 'LTC/BTC', 'profit': -5.01, 'count': 101}, + {'pair': 'TKN/BTC', 'profit': -5.01, 'count': 2}, + {'pair': 'ETH/BTC', 'profit': -5.01, 'count': 100}], + ['TKN/BTC', 'ETH/BTC', 'LTC/BTC']), + # Tie in performance and count, broken by alphabetical sort + ([{"method": "StaticPairList"}, {"method": "PerformanceFilter"}], + ['ETH/BTC', 'TKN/BTC', 'LTC/BTC'], + [{'pair': 'LTC/BTC', 'profit': -5.01, 'count': 1}, + {'pair': 'TKN/BTC', 'profit': -5.01, 'count': 1}, + {'pair': 'ETH/BTC', 'profit': -5.01, 'count': 1}], + ['ETH/BTC', 'LTC/BTC', 'TKN/BTC']), +]) +def test_performance_filter(mocker, whitelist_conf, pairlists, pair_allowlist, overall_performance, + allowlist_result, tickers, markets, ohlcv_history_list): + allowlist_conf = whitelist_conf + allowlist_conf['pairlists'] = pairlists + allowlist_conf['exchange']['pair_whitelist'] = pair_allowlist + + mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) + + freqtrade = get_patched_freqtradebot(mocker, allowlist_conf) + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_tickers=tickers, + markets=PropertyMock(return_value=markets) + ) + mocker.patch.multiple('freqtrade.exchange.Exchange', + get_historic_ohlcv=MagicMock(return_value=ohlcv_history_list), + ) + mocker.patch.multiple('freqtrade.persistence.Trade', + get_overall_performance=MagicMock(return_value=overall_performance), + ) + freqtrade.pairlists.refresh_pairlist() + allowlist = freqtrade.pairlists.whitelist + assert allowlist == allowlist_result diff --git a/tests/pairlist/test_pairlocks.py b/tests/plugins/test_pairlocks.py similarity index 70% rename from tests/pairlist/test_pairlocks.py rename to tests/plugins/test_pairlocks.py index 0b6b89717..bd103b21e 100644 --- a/tests/pairlist/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -79,4 +79,38 @@ def test_PairLocks(use_db): # Nothing was pushed to the database assert len(PairLock.query.all()) == 0 # Reset use-db variable + PairLocks.reset_locks() + PairLocks.use_db = True + + +@pytest.mark.parametrize('use_db', (False, True)) +@pytest.mark.usefixtures("init_persistence") +def test_PairLocks_getlongestlock(use_db): + PairLocks.timeframe = '5m' + # No lock should be present + if use_db: + assert len(PairLock.query.all()) == 0 + else: + PairLocks.use_db = False + + assert PairLocks.use_db == use_db + + pair = 'ETH/BTC' + assert not PairLocks.is_pair_locked(pair) + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime) + # ETH/BTC locked for 4 minutes + assert PairLocks.is_pair_locked(pair) + lock = PairLocks.get_pair_longest_lock(pair) + + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=3) + assert lock.lock_end_time.replace(tzinfo=timezone.utc) < arrow.utcnow().shift(minutes=14) + + PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=15).datetime) + assert PairLocks.is_pair_locked(pair) + + lock = PairLocks.get_pair_longest_lock(pair) + # Must be longer than above + assert lock.lock_end_time.replace(tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=14) + + PairLocks.reset_locks() PairLocks.use_db = True diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py new file mode 100644 index 000000000..e36900a96 --- /dev/null +++ b/tests/plugins/test_protections.py @@ -0,0 +1,412 @@ +import random +from datetime import datetime, timedelta + +import pytest + +from freqtrade import constants +from freqtrade.persistence import PairLocks, Trade +from freqtrade.plugins.protectionmanager import ProtectionManager +from freqtrade.strategy.interface import SellType +from tests.conftest import get_patched_freqtradebot, log_has_re + + +def generate_mock_trade(pair: str, fee: float, is_open: bool, + sell_reason: str = SellType.SELL_SIGNAL, + min_ago_open: int = None, min_ago_close: int = None, + profit_rate: float = 0.9 + ): + open_rate = random.random() + + trade = Trade( + pair=pair, + stake_amount=0.01, + fee_open=fee, + fee_close=fee, + open_date=datetime.utcnow() - timedelta(minutes=min_ago_open or 200), + close_date=datetime.utcnow() - timedelta(minutes=min_ago_close or 30), + open_rate=open_rate, + is_open=is_open, + amount=0.01 / open_rate, + exchange='bittrex', + ) + trade.recalc_open_trade_value() + if not is_open: + trade.close(open_rate * profit_rate) + trade.sell_reason = sell_reason + + return trade + + +def test_protectionmanager(mocker, default_conf): + default_conf['protections'] = [{'method': protection} + for protection in constants.AVAILABLE_PROTECTIONS] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + for handler in freqtrade.protections._protection_handlers: + assert handler.name in constants.AVAILABLE_PROTECTIONS + if not handler.has_global_stop: + assert handler.global_stop(datetime.utcnow()) == (False, None, None) + if not handler.has_local_stop: + assert handler.stop_per_pair('XRP/BTC', datetime.utcnow()) == (False, None, None) + + +@pytest.mark.parametrize('timeframe,expected,protconf', [ + ('1m', [20, 10], + [{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 10}]), + ('5m', [100, 15], + [{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 15}]), + ('1h', [1200, 40], + [{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 40}]), + ('1d', [1440, 5], + [{"method": "StoplossGuard", "lookback_period_candles": 1, "stop_duration": 5}]), + ('1m', [20, 5], + [{"method": "StoplossGuard", "lookback_period": 20, "stop_duration_candles": 5}]), + ('5m', [15, 25], + [{"method": "StoplossGuard", "lookback_period": 15, "stop_duration_candles": 5}]), + ('1h', [50, 600], + [{"method": "StoplossGuard", "lookback_period": 50, "stop_duration_candles": 10}]), + ('1h', [60, 540], + [{"method": "StoplossGuard", "lookback_period_candles": 1, "stop_duration_candles": 9}]), +]) +def test_protections_init(mocker, default_conf, timeframe, expected, protconf): + default_conf['timeframe'] = timeframe + default_conf['protections'] = protconf + man = ProtectionManager(default_conf) + assert len(man._protection_handlers) == len(protconf) + assert man._protection_handlers[0]._lookback_period == expected[0] + assert man._protection_handlers[0]._stop_duration == expected[1] + + +@pytest.mark.usefixtures("init_persistence") +def test_stoploss_guard(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "StoplossGuard", + "lookback_period": 60, + "stop_duration": 40, + "trade_limit": 2 + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, + )) + + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + # This trade does not count, as it's closed too long ago + Trade.session.add(generate_mock_trade( + 'BCH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=250, min_ago_close=100, + )) + + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=240, min_ago_close=30, + )) + # 3 Trades closed - but the 2nd has been closed too long ago. + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'LTC/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=180, min_ago_close=30, + )) + + assert freqtrade.protections.global_stop() + assert log_has_re(message, caplog) + assert PairLocks.is_global_lock() + + # Test 5m after lock-period - this should try and relock the pair, but end-time + # should be the previous end-time + end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5) + assert freqtrade.protections.global_stop(end_time) + assert not PairLocks.is_global_lock(end_time) + + +@pytest.mark.parametrize('only_per_pair', [False, True]) +@pytest.mark.usefixtures("init_persistence") +def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair): + default_conf['protections'] = [{ + "method": "StoplossGuard", + "lookback_period": 60, + "trade_limit": 1, + "stop_duration": 60, + "only_per_pair": only_per_pair + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + pair = 'XRP/BTC' + assert not freqtrade.protections.stop_per_pair(pair) + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, profit_rate=0.9, + )) + + assert not freqtrade.protections.stop_per_pair(pair) + assert not freqtrade.protections.global_stop() + assert not log_has_re(message, caplog) + caplog.clear() + # This trade does not count, as it's closed too long ago + Trade.session.add(generate_mock_trade( + pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=250, min_ago_close=100, profit_rate=0.9, + )) + # Trade does not count for per pair stop as it's the wrong pair. + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=240, min_ago_close=30, profit_rate=0.9, + )) + # 3 Trades closed - but the 2nd has been closed too long ago. + assert not freqtrade.protections.stop_per_pair(pair) + assert freqtrade.protections.global_stop() != only_per_pair + if not only_per_pair: + assert log_has_re(message, caplog) + else: + assert not log_has_re(message, caplog) + + caplog.clear() + + # 2nd Trade that counts with correct pair + Trade.session.add(generate_mock_trade( + pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=180, min_ago_close=30, profit_rate=0.9, + )) + + assert freqtrade.protections.stop_per_pair(pair) + assert freqtrade.protections.global_stop() != only_per_pair + assert PairLocks.is_pair_locked(pair) + assert PairLocks.is_global_lock() != only_per_pair + + +@pytest.mark.usefixtures("init_persistence") +def test_CooldownPeriod(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "CooldownPeriod", + "stop_duration": 60, + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=30, + )) + + assert not freqtrade.protections.global_stop() + assert freqtrade.protections.stop_per_pair('XRP/BTC') + assert PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=205, min_ago_close=35, + )) + + assert not freqtrade.protections.global_stop() + assert not PairLocks.is_pair_locked('ETH/BTC') + assert freqtrade.protections.stop_per_pair('ETH/BTC') + assert PairLocks.is_pair_locked('ETH/BTC') + assert not PairLocks.is_global_lock() + + +@pytest.mark.usefixtures("init_persistence") +def test_LowProfitPairs(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "LowProfitPairs", + "lookback_period": 400, + "stop_duration": 60, + "trade_limit": 2, + "required_profit": 0.0, + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to .*" + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + + assert not log_has_re(message, caplog) + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=800, min_ago_close=450, profit_rate=0.9, + )) + + # Not locked with 1 trade + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=200, min_ago_close=120, profit_rate=0.9, + )) + + # Not locked with 1 trade (first trade is outside of lookback_period) + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + # Add positive trade + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=20, min_ago_close=10, profit_rate=1.15, + )) + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=110, min_ago_close=20, profit_rate=0.8, + )) + + # Locks due to 2nd trade + assert not freqtrade.protections.global_stop() + assert freqtrade.protections.stop_per_pair('XRP/BTC') + assert PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + +@pytest.mark.usefixtures("init_persistence") +def test_MaxDrawdown(mocker, default_conf, fee, caplog): + default_conf['protections'] = [{ + "method": "MaxDrawdown", + "lookback_period": 1000, + "stop_duration": 60, + "trade_limit": 3, + "max_allowed_drawdown": 0.15 + }] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + message = r"Trading stopped due to Max.*" + + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + caplog.clear() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1000, min_ago_close=900, profit_rate=1.1, + )) + Trade.session.add(generate_mock_trade( + 'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1000, min_ago_close=900, profit_rate=1.1, + )) + Trade.session.add(generate_mock_trade( + 'NEO/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1000, min_ago_close=900, profit_rate=1.1, + )) + # No losing trade yet ... so max_drawdown will raise exception + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=500, min_ago_close=400, profit_rate=0.9, + )) + # Not locked with one trade + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value, + min_ago_open=1200, min_ago_close=1100, profit_rate=0.5, + )) + + # Not locked with 1 trade (2nd trade is outside of lookback_period) + assert not freqtrade.protections.global_stop() + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + assert not PairLocks.is_pair_locked('XRP/BTC') + assert not PairLocks.is_global_lock() + assert not log_has_re(message, caplog) + + # Winning trade ... (should not lock, does not change drawdown!) + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=320, min_ago_close=410, profit_rate=1.5, + )) + assert not freqtrade.protections.global_stop() + assert not PairLocks.is_global_lock() + + caplog.clear() + + # Add additional negative trade, causing a loss of > 15% + Trade.session.add(generate_mock_trade( + 'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value, + min_ago_open=20, min_ago_close=10, profit_rate=0.8, + )) + assert not freqtrade.protections.stop_per_pair('XRP/BTC') + # local lock not supported + assert not PairLocks.is_pair_locked('XRP/BTC') + assert freqtrade.protections.global_stop() + assert PairLocks.is_global_lock() + assert log_has_re(message, caplog) + + +@pytest.mark.parametrize("protectionconf,desc_expected,exception_expected", [ + ({"method": "StoplossGuard", "lookback_period": 60, "trade_limit": 2, "stop_duration": 60}, + "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " + "2 stoplosses within 60 minutes.'}]", + None + ), + ({"method": "CooldownPeriod", "stop_duration": 60}, + "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 60 minutes.'}]", + None + ), + ({"method": "LowProfitPairs", "lookback_period": 60, "stop_duration": 60}, + "[{'LowProfitPairs': 'LowProfitPairs - Low Profit Protection, locks pairs with " + "profit < 0.0 within 60 minutes.'}]", + None + ), + ({"method": "MaxDrawdown", "lookback_period": 60, "stop_duration": 60}, + "[{'MaxDrawdown': 'MaxDrawdown - Max drawdown protection, stop trading if drawdown is > 0.0 " + "within 60 minutes.'}]", + None + ), + ({"method": "StoplossGuard", "lookback_period_candles": 12, "trade_limit": 2, + "stop_duration": 60}, + "[{'StoplossGuard': 'StoplossGuard - Frequent Stoploss Guard, " + "2 stoplosses within 12 candles.'}]", + None + ), + ({"method": "CooldownPeriod", "stop_duration_candles": 5}, + "[{'CooldownPeriod': 'CooldownPeriod - Cooldown period of 5 candles.'}]", + None + ), + ({"method": "LowProfitPairs", "lookback_period_candles": 11, "stop_duration": 60}, + "[{'LowProfitPairs': 'LowProfitPairs - Low Profit Protection, locks pairs with " + "profit < 0.0 within 11 candles.'}]", + None + ), + ({"method": "MaxDrawdown", "lookback_period_candles": 20, "stop_duration": 60}, + "[{'MaxDrawdown': 'MaxDrawdown - Max drawdown protection, stop trading if drawdown is > 0.0 " + "within 20 candles.'}]", + None + ), +]) +def test_protection_manager_desc(mocker, default_conf, protectionconf, + desc_expected, exception_expected): + + default_conf['protections'] = [protectionconf] + freqtrade = get_patched_freqtradebot(mocker, default_conf) + + short_desc = str(freqtrade.protections.short_desc()) + assert short_desc == desc_expected diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 47e0f763d..19788c067 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -62,7 +62,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'fee_close_cost': ANY, 'fee_close_currency': ANY, 'open_rate_requested': ANY, - 'open_trade_price': 0.0010025, + 'open_trade_value': 0.0010025, 'close_rate_requested': ANY, 'sell_reason': ANY, 'sell_order_status': ANY, @@ -127,7 +127,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'fee_close_cost': ANY, 'fee_close_currency': ANY, 'open_rate_requested': ANY, - 'open_trade_price': ANY, + 'open_trade_value': ANY, 'close_rate_requested': ANY, 'sell_reason': ANY, 'sell_order_status': ANY, @@ -185,7 +185,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: fetch_ticker=ticker, get_fee=fee, ) - + del default_conf['fiat_display_currency'] freqtradebot = get_patched_freqtradebot(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) rpc = RPC(freqtradebot) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 0dc43474f..e7eee6f05 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -13,6 +13,7 @@ from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade +from freqtrade.rpc import RPC from freqtrade.rpc.api_server import BASE_URI, ApiServer from freqtrade.state import RunMode, State from tests.conftest import create_mock_trades, get_patched_freqtradebot, log_has, patch_get_signal @@ -36,8 +37,9 @@ def botclient(default_conf, mocker): }}) ftbot = get_patched_freqtradebot(mocker, default_conf) + rpc = RPC(ftbot) mocker.patch('freqtrade.rpc.api_server.ApiServer.run', MagicMock()) - apiserver = ApiServer(ftbot) + apiserver = ApiServer(rpc, default_conf) yield ftbot, apiserver.app.test_client() # Cleanup ... ? @@ -179,8 +181,7 @@ def test_api__init__(default_conf, mocker): }}) mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) mocker.patch('freqtrade.rpc.api_server.ApiServer.run', MagicMock()) - - apiserver = ApiServer(get_patched_freqtradebot(mocker, default_conf)) + apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) assert apiserver._config == default_conf @@ -197,7 +198,7 @@ def test_api_run(default_conf, mocker, caplog): server_mock = MagicMock() mocker.patch('freqtrade.rpc.api_server.make_server', server_mock) - apiserver = ApiServer(get_patched_freqtradebot(mocker, default_conf)) + apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) assert apiserver._config == default_conf apiserver.run() @@ -251,7 +252,7 @@ def test_api_cleanup(default_conf, mocker, caplog): mocker.patch('freqtrade.rpc.api_server.threading.Thread', MagicMock()) mocker.patch('freqtrade.rpc.api_server.make_server', MagicMock()) - apiserver = ApiServer(get_patched_freqtradebot(mocker, default_conf)) + apiserver = ApiServer(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf) apiserver.run() stop_mock = MagicMock() stop_mock.shutdown = MagicMock() @@ -559,6 +560,35 @@ def test_api_profit(botclient, mocker, ticker, fee, markets, limit_buy_order, li } +@pytest.mark.usefixtures("init_persistence") +def test_api_stats(botclient, mocker, ticker, fee, markets,): + ftbot, client = botclient + patch_get_signal(ftbot, (True, False)) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + get_balances=MagicMock(return_value=ticker), + fetch_ticker=ticker, + get_fee=fee, + markets=PropertyMock(return_value=markets) + ) + + rc = client_get(client, f"{BASE_URI}/stats") + assert_response(rc, 200) + assert 'durations' in rc.json + assert 'sell_reasons' in rc.json + + create_mock_trades(fee) + + rc = client_get(client, f"{BASE_URI}/stats") + assert_response(rc, 200) + assert 'durations' in rc.json + assert 'sell_reasons' in rc.json + + assert 'wins' in rc.json['durations'] + assert 'losses' in rc.json['durations'] + assert 'draws' in rc.json['durations'] + + def test_api_performance(botclient, mocker, ticker, fee): ftbot, client = botclient patch_get_signal(ftbot, (True, False)) @@ -678,7 +708,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'min_rate': 1.098e-05, 'open_order_id': None, 'open_rate_requested': 1.098e-05, - 'open_trade_price': 0.0010025, + 'open_trade_value': 0.0010025, 'sell_reason': None, 'sell_order_status': None, 'strategy': 'DefaultStrategy', @@ -805,7 +835,7 @@ def test_api_forcebuy(botclient, mocker, fee): 'min_rate': None, 'open_order_id': '123456', 'open_rate_requested': None, - 'open_trade_price': 0.24605460, + 'open_trade_value': 0.24605460, 'sell_reason': None, 'sell_order_status': None, 'strategy': None, @@ -841,7 +871,7 @@ def test_api_forcesell(botclient, mocker, ticker, fee, markets): def test_api_pair_candles(botclient, ohlcv_history): ftbot, client = botclient timeframe = '5m' - amount = 2 + amount = 3 # No pair rc = client_get(client, @@ -881,8 +911,8 @@ def test_api_pair_candles(botclient, ohlcv_history): assert 'data_stop_ts' in rc.json assert rc.json['data_start'] == '2017-11-26 08:50:00+00:00' assert rc.json['data_start_ts'] == 1511686200000 - assert rc.json['data_stop'] == '2017-11-26 08:55:00+00:00' - assert rc.json['data_stop_ts'] == 1511686500000 + assert rc.json['data_stop'] == '2017-11-26 09:00:00+00:00' + assert rc.json['data_stop_ts'] == 1511686800000 assert isinstance(rc.json['columns'], list) assert rc.json['columns'] == ['date', 'open', 'high', 'low', 'close', 'volume', 'sma', 'buy', 'sell', @@ -897,7 +927,10 @@ def test_api_pair_candles(botclient, ohlcv_history): [['2017-11-26 08:50:00', 8.794e-05, 8.948e-05, 8.794e-05, 8.88e-05, 0.0877869, None, 0, 0, 1511686200000, None, None], ['2017-11-26 08:55:00', 8.88e-05, 8.942e-05, 8.88e-05, - 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 1511686500000, 8.88e-05, None] + 8.893e-05, 0.05874751, 8.886500000000001e-05, 1, 0, 1511686500000, 8.88e-05, None], + ['2017-11-26 09:00:00', 8.891e-05, 8.893e-05, 8.875e-05, 8.877e-05, + 0.7039405, 8.885000000000002e-05, 0, 0, 1511686800000, None, None] + ]) diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index 4b715fc37..06706120f 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -137,7 +137,7 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: freqtradebot = get_patched_freqtradebot(mocker, default_conf) rpc_manager = RPCManager(freqtradebot) - rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists, freqtradebot.protections) assert telegram_mock.call_count == 3 assert "*Exchange:* `bittrex`" in telegram_mock.call_args_list[1][0][0]['status'] @@ -147,10 +147,14 @@ def test_startupmessages_telegram_enabled(mocker, default_conf, caplog) -> None: default_conf['whitelist'] = {'method': 'VolumePairList', 'config': {'number_assets': 20} } + default_conf['protections'] = [{"method": "StoplossGuard", + "lookback_period": 60, "trade_limit": 2, "stop_duration": 60}] + freqtradebot = get_patched_freqtradebot(mocker, default_conf) - rpc_manager.startup_messages(default_conf, freqtradebot.pairlists) - assert telegram_mock.call_count == 3 + rpc_manager.startup_messages(default_conf, freqtradebot.pairlists, freqtradebot.protections) + assert telegram_mock.call_count == 4 assert "Dry run is enabled." in telegram_mock.call_args_list[0][0][0]['status'] + assert 'StoplossGuard' in telegram_mock.call_args_list[-1][0][0]['status'] def test_init_apiserver_disabled(mocker, default_conf, caplog) -> None: diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index ace44a34a..97b9e5e7c 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -6,20 +6,21 @@ import re from datetime import datetime from random import choice, randint from string import ascii_uppercase -from unittest.mock import ANY, MagicMock, PropertyMock +from unittest.mock import ANY, MagicMock import arrow import pytest -from telegram import Chat, Message, Update +from telegram import Chat, Message, ReplyKeyboardMarkup, Update from telegram.error import NetworkError from freqtrade import __version__ from freqtrade.constants import CANCEL_REASON from freqtrade.edge import PairInfo +from freqtrade.exceptions import OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.loggers import setup_logging from freqtrade.persistence import PairLocks, Trade -from freqtrade.rpc import RPCMessageType +from freqtrade.rpc import RPC, RPCMessageType from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import RunMode, State from freqtrade.strategy.interface import SellType @@ -31,8 +32,8 @@ class DummyCls(Telegram): """ Dummy class for testing the Telegram @authorized_only decorator """ - def __init__(self, freqtrade) -> None: - super().__init__(freqtrade) + def __init__(self, rpc: RPC, config) -> None: + super().__init__(rpc, config) self.state = {'called': False} def _init(self): @@ -53,12 +54,27 @@ class DummyCls(Telegram): raise Exception('test') -def test__init__(default_conf, mocker) -> None: +def get_telegram_testobject(mocker, default_conf, mock=True, ftbot=None): + msg_mock = MagicMock() + if mock: + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _init=MagicMock(), + _send_msg=msg_mock + ) + if not ftbot: + ftbot = get_patched_freqtradebot(mocker, default_conf) + rpc = RPC(ftbot) + telegram = Telegram(rpc, default_conf) + + return telegram, ftbot, msg_mock + + +def test_telegram__init__(default_conf, mocker) -> None: mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) - assert telegram._updater is None + telegram, _, _ = get_telegram_testobject(mocker, default_conf) assert telegram._config == default_conf @@ -66,7 +82,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: start_polling = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', MagicMock(return_value=start_polling)) - Telegram(get_patched_freqtradebot(mocker, default_conf)) + get_telegram_testobject(mocker, default_conf, mock=False) assert start_polling.call_count == 0 # number of handles registered @@ -75,9 +91,10 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: message_str = ("rpc.telegram is listening for following commands: [['status'], ['profit'], " "['balance'], ['start'], ['stop'], ['forcesell'], ['forcebuy'], ['trades'], " - "['delete'], ['performance'], ['daily'], ['count'], ['locks'], " + "['delete'], ['performance'], ['stats'], ['daily'], ['count'], ['locks'], " "['reload_config', 'reload_conf'], ['show_config', 'show_conf'], ['stopbuy'], " - "['whitelist'], ['blacklist'], ['logs'], ['edge'], ['help'], ['version']]") + "['whitelist'], ['blacklist'], ['logs'], ['edge'], ['help'], ['version']" + "]") assert log_has(message_str, caplog) @@ -87,7 +104,7 @@ def test_cleanup(default_conf, mocker, ) -> None: updater_mock.stop = MagicMock() mocker.patch('freqtrade.rpc.telegram.Updater', updater_mock) - telegram = Telegram(get_patched_freqtradebot(mocker, default_conf)) + telegram, _, _ = get_telegram_testobject(mocker, default_conf, mock=False) telegram.cleanup() assert telegram._updater.stop.call_count == 1 @@ -97,8 +114,10 @@ def test_authorized_only(default_conf, mocker, caplog, update) -> None: default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) + rpc = RPC(bot) + dummy = DummyCls(rpc, default_conf) + patch_get_signal(bot, (True, False)) - dummy = DummyCls(bot) dummy.dummy_handler(update=update, context=MagicMock()) assert dummy.state['called'] is True assert log_has('Executing handler: dummy_handler for chat_id: 0', caplog) @@ -114,8 +133,10 @@ def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) + rpc = RPC(bot) + dummy = DummyCls(rpc, default_conf) + patch_get_signal(bot, (True, False)) - dummy = DummyCls(bot) dummy.dummy_handler(update=update, context=MagicMock()) assert dummy.state['called'] is False assert not log_has('Executing handler: dummy_handler for chat_id: 3735928559', caplog) @@ -129,8 +150,9 @@ def test_authorized_only_exception(default_conf, mocker, caplog, update) -> None default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) + rpc = RPC(bot) + dummy = DummyCls(rpc, default_conf) patch_get_signal(bot, (True, False)) - dummy = DummyCls(bot) dummy.dummy_exception(update=update, context=MagicMock()) assert dummy.state['called'] is False @@ -144,11 +166,11 @@ def test_telegram_status(default_conf, update, mocker) -> None: default_conf['telegram']['enabled'] = False default_conf['telegram']['chat_id'] = "123" - msg_mock = MagicMock() status_table = MagicMock() + mocker.patch('freqtrade.rpc.telegram.Telegram._status_table', status_table) + mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), + 'freqtrade.rpc.rpc.RPC', _rpc_trade_status=MagicMock(return_value=[{ 'trade_id': 1, 'pair': 'ETH/BTC', @@ -175,12 +197,9 @@ def test_telegram_status(default_conf, update, mocker) -> None: 'open_order': '(limit buy rem=0.00000000)', 'is_open': True }]), - _status_table=status_table, - _send_msg=msg_mock ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._status(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -198,21 +217,16 @@ def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: fetch_ticker=ticker, get_fee=fee, ) - msg_mock = MagicMock() status_table = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), _status_table=status_table, - _send_msg=msg_mock ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - freqtradebot.state = State.STOPPED # Status is also enabled when stopped telegram._status(update=update, context=MagicMock()) @@ -248,18 +262,12 @@ def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: fetch_ticker=ticker, get_fee=fee, ) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) default_conf['stake_amount'] = 15.0 - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + + patch_get_signal(freqtradebot, (True, False)) freqtradebot.state = State.STOPPED # Status table is also enabled when stopped @@ -300,16 +308,10 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, fetch_ticker=ticker, get_fee=fee, ) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Create some test data freqtradebot.enter_positions() @@ -338,6 +340,18 @@ def test_daily_handle(default_conf, update, ticker, limit_buy_order, fee, assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] + # Reset msg_mock + msg_mock.reset_mock() + context.args = [] + telegram._daily(update=update, context=context) + assert msg_mock.call_count == 1 + assert 'Daily' in msg_mock.call_args_list[0][0][0] + assert str(datetime.utcnow().date()) in msg_mock.call_args_list[0][0][0] + assert str(' 0.00006217 BTC') in msg_mock.call_args_list[0][0][0] + assert str(' 0.933 USD') in msg_mock.call_args_list[0][0][0] + assert str(' 1 trade') in msg_mock.call_args_list[0][0][0] + assert str(' 0 trade') in msg_mock.call_args_list[0][0][0] + # Reset msg_mock msg_mock.reset_mock() freqtradebot.config['max_open_trades'] = 2 @@ -366,16 +380,9 @@ def test_daily_wrong_input(default_conf, update, ticker, mocker) -> None: 'freqtrade.exchange.Exchange', fetch_ticker=ticker ) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Try invalid data msg_mock.reset_mock() @@ -405,16 +412,9 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, fetch_ticker=ticker, get_fee=fee, ) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) telegram._profit(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -457,6 +457,33 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, assert '*Best Performing:* `ETH/BTC: 6.20%`' in msg_mock.call_args_list[-1][0][0] +def test_telegram_stats(default_conf, update, ticker, ticker_sell_up, fee, + limit_buy_order, limit_sell_order, mocker) -> None: + mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) + mocker.patch.multiple( + 'freqtrade.exchange.Exchange', + fetch_ticker=ticker, + get_fee=fee, + ) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + patch_get_signal(freqtradebot, (True, False)) + + telegram._stats(update=update, context=MagicMock()) + assert msg_mock.call_count == 1 + # assert 'No trades yet.' in msg_mock.call_args_list[0][0][0] + msg_mock.reset_mock() + + # Create some test data + create_mock_trades(fee) + + telegram._stats(update=update, context=MagicMock()) + assert msg_mock.call_count == 1 + assert 'Sell Reason' in msg_mock.call_args_list[-1][0][0] + assert 'ROI' in msg_mock.call_args_list[-1][0][0] + assert 'Avg. Duration' in msg_mock.call_args_list[-1][0][0] + msg_mock.reset_mock() + + def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tickers) -> None: default_conf['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value=rpc_balance) @@ -464,18 +491,9 @@ def test_telegram_balance_handle(default_conf, update, mocker, rpc_balance, tick mocker.patch('freqtrade.exchange.Exchange.get_valid_pair_combination', side_effect=lambda a, b: f"{a}/{b}") - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 @@ -493,18 +511,9 @@ def test_balance_handle_empty_response(default_conf, update, mocker) -> None: default_conf['dry_run'] = False mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={}) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - freqtradebot.config['dry_run'] = False telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] @@ -515,18 +524,9 @@ def test_balance_handle_empty_response(default_conf, update, mocker) -> None: def test_balance_handle_empty_response_dry(default_conf, update, mocker) -> None: mocker.patch('freqtrade.exchange.Exchange.get_balances', return_value={}) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - telegram._balance(update=update, context=MagicMock()) result = msg_mock.call_args_list[0][0][0] assert msg_mock.call_count == 1 @@ -553,18 +553,9 @@ def test_balance_handle_too_large_response(default_conf, update, mocker) -> None 'value': 1000.0, }) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) - telegram._balance(update=update, context=MagicMock()) assert msg_mock.call_count > 1 # Test if wrap happens around 4000 - @@ -575,15 +566,8 @@ def test_balance_handle_too_large_response(default_conf, update, mocker) -> None def test_start_handle(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.STOPPED assert freqtradebot.state == State.STOPPED @@ -593,15 +577,8 @@ def test_start_handle(default_conf, update, mocker) -> None: def test_start_handle_already_running(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.RUNNING assert freqtradebot.state == State.RUNNING @@ -612,15 +589,8 @@ def test_start_handle_already_running(default_conf, update, mocker) -> None: def test_stop_handle(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.RUNNING assert freqtradebot.state == State.RUNNING @@ -631,15 +601,8 @@ def test_stop_handle(default_conf, update, mocker) -> None: def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.STOPPED assert freqtradebot.state == State.STOPPED @@ -650,15 +613,8 @@ def test_stop_handle_already_stopped(default_conf, update, mocker) -> None: def test_stopbuy_handle(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) assert freqtradebot.config['max_open_trades'] != 0 telegram._stopbuy(update=update, context=MagicMock()) @@ -669,15 +625,8 @@ def test_stopbuy_handle(default_conf, update, mocker) -> None: def test_reload_config_handle(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) freqtradebot.state = State.RUNNING assert freqtradebot.state == State.RUNNING @@ -690,7 +639,7 @@ def test_reload_config_handle(default_conf, update, mocker) -> None: def test_telegram_forcesell_handle(default_conf, update, ticker, fee, ticker_sell_up, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) - rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + msg_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_exchange(mocker) patch_whitelist(mocker, default_conf) @@ -701,8 +650,9 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, ) freqtradebot = FreqtradeBot(default_conf) + rpc = RPC(freqtradebot) + telegram = Telegram(rpc, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Create some test data freqtradebot.enter_positions() @@ -718,8 +668,8 @@ def test_telegram_forcesell_handle(default_conf, update, ticker, fee, context.args = ["1"] telegram._forcesell(update=update, context=context) - assert rpc_mock.call_count == 3 - last_msg = rpc_mock.call_args_list[-1][0][0] + assert msg_mock.call_count == 3 + last_msg = msg_mock.call_args_list[-1][0][0] assert { 'type': RPCMessageType.SELL_NOTIFICATION, 'trade_id': 1, @@ -745,7 +695,7 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, ticker_sell_down, mocker) -> None: mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + msg_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_exchange(mocker) patch_whitelist(mocker, default_conf) @@ -757,8 +707,9 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, ) freqtradebot = FreqtradeBot(default_conf) + rpc = RPC(freqtradebot) + telegram = Telegram(rpc, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Create some test data freqtradebot.enter_positions() @@ -777,9 +728,9 @@ def test_telegram_forcesell_down_handle(default_conf, update, ticker, fee, context.args = ["1"] telegram._forcesell(update=update, context=context) - assert rpc_mock.call_count == 3 + assert msg_mock.call_count == 3 - last_msg = rpc_mock.call_args_list[-1][0][0] + last_msg = msg_mock.call_args_list[-1][0][0] assert { 'type': RPCMessageType.SELL_NOTIFICATION, 'trade_id': 1, @@ -805,7 +756,7 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None patch_exchange(mocker) mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) + msg_mock = mocker.patch('freqtrade.rpc.telegram.Telegram.send_msg', MagicMock()) mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) patch_whitelist(mocker, default_conf) mocker.patch.multiple( @@ -815,12 +766,13 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None ) default_conf['max_open_trades'] = 4 freqtradebot = FreqtradeBot(default_conf) + rpc = RPC(freqtradebot) + telegram = Telegram(rpc, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Create some test data freqtradebot.enter_positions() - rpc_mock.reset_mock() + msg_mock.reset_mock() # /forcesell all context = MagicMock() @@ -828,8 +780,8 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None telegram._forcesell(update=update, context=context) # Called for each trade 3 times - assert rpc_mock.call_count == 8 - msg = rpc_mock.call_args_list[1][0][0] + assert msg_mock.call_count == 8 + msg = msg_mock.call_args_list[1][0][0] assert { 'type': RPCMessageType.SELL_NOTIFICATION, 'trade_id': 1, @@ -854,16 +806,9 @@ def test_forcesell_all_handle(default_conf, update, ticker, fee, mocker) -> None def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: mocker.patch('freqtrade.rpc.fiat_convert.CryptoToFiatConverter._find_price', return_value=15000.0) - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Trader is not running freqtradebot.state = State.STOPPED @@ -881,7 +826,7 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: context.args = [] telegram._forcesell(update=update, context=context) assert msg_mock.call_count == 1 - assert 'invalid argument' in msg_mock.call_args_list[0][0][0] + assert "You must specify a trade-id or 'all'." in msg_mock.call_args_list[0][0][0] # Invalid argument msg_mock.reset_mock() @@ -894,21 +839,14 @@ def test_forcesell_handle_invalid(default_conf, update, mocker) -> None: assert 'invalid argument' in msg_mock.call_args_list[0][0][0] -def test_forcebuy_handle(default_conf, update, markets, mocker) -> None: +def test_forcebuy_handle(default_conf, update, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) - mocker.patch('freqtrade.rpc.telegram.Telegram._send_msg', MagicMock()) - mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - markets=PropertyMock(markets), - ) + fbuy_mock = MagicMock(return_value=None) mocker.patch('freqtrade.rpc.RPC._rpc_forcebuy', fbuy_mock) - freqtradebot = FreqtradeBot(default_conf) + telegram, freqtradebot, _ = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # /forcebuy ETH/BTC context = MagicMock() @@ -933,42 +871,29 @@ def test_forcebuy_handle(default_conf, update, markets, mocker) -> None: assert fbuy_mock.call_args_list[0][0][1] == 0.055 -def test_forcebuy_handle_exception(default_conf, update, markets, mocker) -> None: +def test_forcebuy_handle_exception(default_conf, update, mocker) -> None: mocker.patch('freqtrade.rpc.rpc.CryptoToFiatConverter._find_price', return_value=15000.0) - rpc_mock = mocker.patch('freqtrade.rpc.telegram.Telegram._send_msg', MagicMock()) - mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock()) - patch_exchange(mocker) - mocker.patch.multiple( - 'freqtrade.exchange.Exchange', - markets=PropertyMock(markets), - ) - freqtradebot = FreqtradeBot(default_conf) + + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) update.message.text = '/forcebuy ETH/Nonepair' telegram._forcebuy(update=update, context=MagicMock()) - assert rpc_mock.call_count == 1 - assert rpc_mock.call_args_list[0][0][0] == 'Forcebuy not enabled.' + assert msg_mock.call_count == 1 + assert msg_mock.call_args_list[0][0][0] == 'Forcebuy not enabled.' def test_performance_handle(default_conf, update, ticker, fee, limit_buy_order, limit_sell_order, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) + mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) # Create some test data freqtradebot.enter_positions() @@ -990,20 +915,13 @@ def test_performance_handle(default_conf, update, ticker, fee, def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) freqtradebot.state = State.STOPPED telegram._count(update=update, context=MagicMock()) @@ -1026,20 +944,13 @@ def test_count_handle(default_conf, update, ticker, fee, mocker) -> None: def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=ticker, get_fee=fee, ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) patch_get_signal(freqtradebot, (True, False)) - telegram = Telegram(freqtradebot) PairLocks.lock_pair('ETH/BTC', arrow.utcnow().shift(minutes=4).datetime, 'randreason') PairLocks.lock_pair('XRP/BTC', arrow.utcnow().shift(minutes=20).datetime, 'deadbeef') @@ -1056,15 +967,8 @@ def test_telegram_lock_handle(default_conf, update, ticker, fee, mocker) -> None def test_whitelist_static(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1073,19 +977,11 @@ def test_whitelist_static(default_conf, update, mocker) -> None: def test_whitelist_dynamic(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) default_conf['pairlists'] = [{'method': 'VolumePairList', 'number_assets': 4 }] - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - - telegram = Telegram(freqtradebot) + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._whitelist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1094,15 +990,8 @@ def test_whitelist_dynamic(default_conf, update, mocker) -> None: def test_blacklist_static(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._blacklist(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1134,17 +1023,14 @@ def test_blacklist_static(default_conf, update, mocker) -> None: def test_telegram_logs(default_conf, update, mocker) -> None: - msg_mock = MagicMock() mocker.patch.multiple( 'freqtrade.rpc.telegram.Telegram', _init=MagicMock(), - _send_msg=msg_mock ) setup_logging(default_conf) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) - telegram = Telegram(freqtradebot) context = MagicMock() context.args = [] telegram._logs(update=update, context=context) @@ -1168,16 +1054,8 @@ def test_telegram_logs(default_conf, update, mocker) -> None: def test_edge_disabled(default_conf, update, mocker) -> None: - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - - telegram = Telegram(freqtradebot) + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) telegram._edge(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1185,21 +1063,13 @@ def test_edge_disabled(default_conf, update, mocker) -> None: def test_edge_enabled(edge_conf, update, mocker) -> None: - msg_mock = MagicMock() mocker.patch('freqtrade.edge.Edge._cached_pairs', mocker.PropertyMock( return_value={ 'E/F': PairInfo(-0.01, 0.66, 3.71, 0.50, 1.71, 10, 60), } )) - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, edge_conf) - - telegram = Telegram(freqtradebot) + telegram, _, msg_mock = get_telegram_testobject(mocker, edge_conf) telegram._edge(update=update, context=MagicMock()) assert msg_mock.call_count == 1 @@ -1208,23 +1078,23 @@ def test_edge_enabled(edge_conf, update, mocker) -> None: def test_telegram_trades(mocker, update, default_conf, fee): - msg_mock = MagicMock() - mocker.patch.multiple( - 'freqtrade.rpc.telegram.Telegram', - _init=MagicMock(), - _send_msg=msg_mock - ) - freqtradebot = get_patched_freqtradebot(mocker, default_conf) - telegram = Telegram(freqtradebot) + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + context = MagicMock() context.args = [] telegram._trades(update=update, context=context) assert "0 recent trades:" in msg_mock.call_args_list[0][0][0] assert "
" not in msg_mock.call_args_list[0][0][0]
-
     msg_mock.reset_mock()
+
+    context.args = ['hello']
+    telegram._trades(update=update, context=context)
+    assert "0 recent trades:" in msg_mock.call_args_list[0][0][0]
+    assert "
" not in msg_mock.call_args_list[0][0][0]
+    msg_mock.reset_mock()
+
     create_mock_trades(fee)
 
     context = MagicMock()
@@ -1238,20 +1108,13 @@ def test_telegram_trades(mocker, update, default_conf, fee):
 
 
 def test_telegram_delete_trade(mocker, update, default_conf, fee):
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
 
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     context = MagicMock()
     context.args = []
 
     telegram._delete_trade(update=update, context=context)
-    assert "invalid argument" in msg_mock.call_args_list[0][0][0]
+    assert "Trade-id not set." in msg_mock.call_args_list[0][0][0]
 
     msg_mock.reset_mock()
     create_mock_trades(fee)
@@ -1265,15 +1128,7 @@ def test_telegram_delete_trade(mocker, update, default_conf, fee):
 
 
 def test_help_handle(default_conf, update, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-
-    telegram = Telegram(freqtradebot)
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram._help(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
@@ -1281,14 +1136,8 @@ def test_help_handle(default_conf, update, mocker) -> None:
 
 
 def test_version_handle(default_conf, update, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram._version(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
@@ -1296,15 +1145,10 @@ def test_version_handle(default_conf, update, mocker) -> None:
 
 
 def test_show_config_handle(default_conf, update, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
+
     default_conf['runmode'] = RunMode.DRY_RUN
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf)
 
     telegram._show_config(update=update, context=MagicMock())
     assert msg_mock.call_count == 1
@@ -1324,12 +1168,7 @@ def test_show_config_handle(default_conf, update, mocker) -> None:
 
 
 def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
+
     msg = {
         'type': RPCMessageType.BUY_NOTIFICATION,
         'exchange': 'Bittrex',
@@ -1344,8 +1183,8 @@ def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None:
         'amount': 1333.3333333333335,
         'open_date': arrow.utcnow().shift(hours=-1)
     }
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf)
+
     telegram.send_msg(msg)
     assert msg_mock.call_args[0][0] \
         == '\N{LARGE BLUE CIRCLE} *Bittrex:* Buying ETH/BTC\n' \
@@ -1371,14 +1210,9 @@ def test_send_msg_buy_notification(default_conf, mocker, caplog) -> None:
 
 
 def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
     telegram.send_msg({
         'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
         'exchange': 'Bittrex',
@@ -1390,16 +1224,11 @@ def test_send_msg_buy_cancel_notification(default_conf, mocker) -> None:
 
 
 def test_send_msg_sell_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
-    old_convamount = telegram._fiat_converter.convert_amount
-    telegram._fiat_converter.convert_amount = lambda a, b, c: -24.812
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
+    old_convamount = telegram._rpc._fiat_converter.convert_amount
+    telegram._rpc._fiat_converter.convert_amount = lambda a, b, c: -24.812
     telegram.send_msg({
         'type': RPCMessageType.SELL_NOTIFICATION,
         'exchange': 'Binance',
@@ -1456,20 +1285,15 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None:
             '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n'
             '*Profit:* `-57.41%`')
     # Reset singleton function to avoid random breaks
-    telegram._fiat_converter.convert_amount = old_convamount
+    telegram._rpc._fiat_converter.convert_amount = old_convamount
 
 
 def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
-    old_convamount = telegram._fiat_converter.convert_amount
-    telegram._fiat_converter.convert_amount = lambda a, b, c: -24.812
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
+    old_convamount = telegram._rpc._fiat_converter.convert_amount
+    telegram._rpc._fiat_converter.convert_amount = lambda a, b, c: -24.812
     telegram.send_msg({
         'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
         'exchange': 'Binance',
@@ -1490,18 +1314,12 @@ def test_send_msg_sell_cancel_notification(default_conf, mocker) -> None:
     assert msg_mock.call_args[0][0] \
         == ('\N{WARNING SIGN} *Binance:* Cancelling Open Sell Order for KEY/ETH. Reason: timeout')
     # Reset singleton function to avoid random breaks
-    telegram._fiat_converter.convert_amount = old_convamount
+    telegram._rpc._fiat_converter.convert_amount = old_convamount
 
 
 def test_send_msg_status_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     telegram.send_msg({
         'type': RPCMessageType.STATUS_NOTIFICATION,
         'status': 'running'
@@ -1510,14 +1328,7 @@ def test_send_msg_status_notification(default_conf, mocker) -> None:
 
 
 def test_warning_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     telegram.send_msg({
         'type': RPCMessageType.WARNING_NOTIFICATION,
         'status': 'message'
@@ -1526,14 +1337,7 @@ def test_warning_notification(default_conf, mocker) -> None:
 
 
 def test_startup_notification(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
     telegram.send_msg({
         'type': RPCMessageType.STARTUP_NOTIFICATION,
         'status': '*Custom:* `Hello World`'
@@ -1542,14 +1346,7 @@ def test_startup_notification(default_conf, mocker) -> None:
 
 
 def test_send_msg_unknown_type(default_conf, mocker) -> None:
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, _ = get_telegram_testobject(mocker, default_conf)
     with pytest.raises(NotImplementedError, match=r'Unknown message type: None'):
         telegram.send_msg({
             'type': None,
@@ -1558,14 +1355,8 @@ def test_send_msg_unknown_type(default_conf, mocker) -> None:
 
 def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None:
     del default_conf['fiat_display_currency']
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
     telegram.send_msg({
         'type': RPCMessageType.BUY_NOTIFICATION,
         'exchange': 'Bittrex',
@@ -1589,14 +1380,8 @@ def test_send_msg_buy_notification_no_fiat(default_conf, mocker) -> None:
 
 def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None:
     del default_conf['fiat_display_currency']
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf)
+
     telegram.send_msg({
         'type': RPCMessageType.SELL_NOTIFICATION,
         'exchange': 'Binance',
@@ -1636,14 +1421,8 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None:
 ])
 def test__sell_emoji(default_conf, mocker, msg, expected):
     del default_conf['fiat_display_currency']
-    msg_mock = MagicMock()
-    mocker.patch.multiple(
-        'freqtrade.rpc.telegram.Telegram',
-        _init=MagicMock(),
-        _send_msg=msg_mock
-    )
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+
+    telegram, _, _ = get_telegram_testobject(mocker, default_conf)
 
     assert telegram._get_sell_emoji(msg) == expected
 
@@ -1651,8 +1430,7 @@ def test__sell_emoji(default_conf, mocker, msg, expected):
 def test__send_msg(default_conf, mocker) -> None:
     mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock())
     bot = MagicMock()
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, _ = get_telegram_testobject(mocker, default_conf, mock=False)
     telegram._updater = MagicMock()
     telegram._updater.bot = bot
 
@@ -1665,8 +1443,7 @@ def test__send_msg_network_error(default_conf, mocker, caplog) -> None:
     mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock())
     bot = MagicMock()
     bot.send_message = MagicMock(side_effect=NetworkError('Oh snap'))
-    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
-    telegram = Telegram(freqtradebot)
+    telegram, _, _ = get_telegram_testobject(mocker, default_conf, mock=False)
     telegram._updater = MagicMock()
     telegram._updater.bot = bot
 
@@ -1676,3 +1453,54 @@ def test__send_msg_network_error(default_conf, mocker, caplog) -> None:
     # Bot should've tried to send it twice
     assert len(bot.method_calls) == 2
     assert log_has('Telegram NetworkError: Oh snap! Trying one more time.', caplog)
+
+
+def test__send_msg_keyboard(default_conf, mocker, caplog) -> None:
+    mocker.patch('freqtrade.rpc.telegram.Telegram._init', MagicMock())
+    bot = MagicMock()
+    bot.send_message = MagicMock()
+    freqtradebot = get_patched_freqtradebot(mocker, default_conf)
+    rpc = RPC(freqtradebot)
+
+    invalid_keys_list = [['/not_valid', '/profit'], ['/daily'], ['/alsoinvalid']]
+    default_keys_list = [['/daily', '/profit', '/balance'],
+                         ['/status', '/status table', '/performance'],
+                         ['/count', '/start', '/stop', '/help']]
+    default_keyboard = ReplyKeyboardMarkup(default_keys_list)
+
+    custom_keys_list = [['/daily', '/stats', '/balance', '/profit'],
+                        ['/count', '/start', '/reload_config', '/help']]
+    custom_keyboard = ReplyKeyboardMarkup(custom_keys_list)
+
+    def init_telegram(freqtradebot):
+        telegram = Telegram(rpc, default_conf)
+        telegram._updater = MagicMock()
+        telegram._updater.bot = bot
+        return telegram
+
+    # no keyboard in config -> default keyboard
+    freqtradebot.config['telegram']['enabled'] = True
+    telegram = init_telegram(freqtradebot)
+    telegram._send_msg('test')
+    used_keyboard = bot.send_message.call_args[1]['reply_markup']
+    assert used_keyboard == default_keyboard
+
+    # invalid keyboard in config -> default keyboard
+    freqtradebot.config['telegram']['enabled'] = True
+    freqtradebot.config['telegram']['keyboard'] = invalid_keys_list
+    err_msg = re.escape("config.telegram.keyboard: Invalid commands for custom "
+                        "Telegram keyboard: ['/not_valid', '/alsoinvalid']"
+                        "\nvalid commands are: ") + r"*"
+    with pytest.raises(OperationalException, match=err_msg):
+        telegram = init_telegram(freqtradebot)
+
+    # valid keyboard in config -> custom keyboard
+    freqtradebot.config['telegram']['enabled'] = True
+    freqtradebot.config['telegram']['keyboard'] = custom_keys_list
+    telegram = init_telegram(freqtradebot)
+    telegram._send_msg('test')
+    used_keyboard = bot.send_message.call_args[1]['reply_markup']
+    assert used_keyboard == custom_keyboard
+    assert log_has("using custom keyboard from config.json: "
+                   "[['/daily', '/stats', '/balance', '/profit'], ['/count', "
+                   "'/start', '/reload_config', '/help']]", caplog)
diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py
index 9256a5316..4ca547390 100644
--- a/tests/rpc/test_rpc_webhook.py
+++ b/tests/rpc/test_rpc_webhook.py
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock
 import pytest
 from requests import RequestException
 
-from freqtrade.rpc import RPCMessageType
+from freqtrade.rpc import RPC, RPCMessageType
 from freqtrade.rpc.webhook import Webhook
 from freqtrade.strategy.interface import SellType
 from tests.conftest import get_patched_freqtradebot, log_has
@@ -45,7 +45,7 @@ def get_webhook_dict() -> dict:
 
 def test__init__(mocker, default_conf):
     default_conf['webhook'] = {'enabled': True, 'url': "https://DEADBEEF.com"}
-    webhook = Webhook(get_patched_freqtradebot(mocker, default_conf))
+    webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf)
     assert webhook._config == default_conf
 
 
@@ -53,7 +53,7 @@ def test_send_msg(default_conf, mocker):
     default_conf["webhook"] = get_webhook_dict()
     msg_mock = MagicMock()
     mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
-    webhook = Webhook(get_patched_freqtradebot(mocker, default_conf))
+    webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf)
     # Test buy
     msg_mock = MagicMock()
     mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
@@ -172,7 +172,7 @@ def test_exception_send_msg(default_conf, mocker, caplog):
     default_conf["webhook"] = get_webhook_dict()
     del default_conf["webhook"]["webhookbuy"]
 
-    webhook = Webhook(get_patched_freqtradebot(mocker, default_conf))
+    webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf)
     webhook.send_msg({'type': RPCMessageType.BUY_NOTIFICATION})
     assert log_has(f"Message type '{RPCMessageType.BUY_NOTIFICATION}' not configured for webhooks",
                    caplog)
@@ -181,7 +181,7 @@ def test_exception_send_msg(default_conf, mocker, caplog):
     default_conf["webhook"]["webhookbuy"]["value1"] = "{DEADBEEF:8f}"
     msg_mock = MagicMock()
     mocker.patch("freqtrade.rpc.webhook.Webhook._send_msg", msg_mock)
-    webhook = Webhook(get_patched_freqtradebot(mocker, default_conf))
+    webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf)
     msg = {
         'type': RPCMessageType.BUY_NOTIFICATION,
         'exchange': 'Bittrex',
@@ -209,7 +209,7 @@ def test_exception_send_msg(default_conf, mocker, caplog):
 
 def test__send_msg(default_conf, mocker, caplog):
     default_conf["webhook"] = get_webhook_dict()
-    webhook = Webhook(get_patched_freqtradebot(mocker, default_conf))
+    webhook = Webhook(RPC(get_patched_freqtradebot(mocker, default_conf)), default_conf)
     msg = {'value1': 'DEADBEEF',
            'value2': 'ALIVEBEEF',
            'value3': 'FREQTRADE'}
diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py
index 7cf9a0624..640849ba4 100644
--- a/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@@ -128,27 +128,29 @@ def test_assert_df_raise(default_conf, mocker, caplog, ohlcv_history):
 
 
 def test_assert_df(default_conf, mocker, ohlcv_history, caplog):
+    df_len = len(ohlcv_history) - 1
     # Ensure it's running when passed correctly
     _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                        ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date'])
+                        ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[df_len, 'date'])
 
     with pytest.raises(StrategyError, match=r"Dataframe returned from strategy.*length\."):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history) + 1,
-                            ohlcv_history.loc[1, 'close'], ohlcv_history.loc[1, 'date'])
+                            ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[df_len, 'date'])
 
     with pytest.raises(StrategyError,
                        match=r"Dataframe returned from strategy.*last close price\."):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                            ohlcv_history.loc[1, 'close'] + 0.01, ohlcv_history.loc[1, 'date'])
+                            ohlcv_history.loc[df_len, 'close'] + 0.01,
+                            ohlcv_history.loc[df_len, 'date'])
     with pytest.raises(StrategyError,
                        match=r"Dataframe returned from strategy.*last date\."):
         _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                            ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date'])
+                            ohlcv_history.loc[df_len, 'close'], ohlcv_history.loc[0, 'date'])
 
     _STRATEGY.disable_dataframe_checks = True
     caplog.clear()
     _STRATEGY.assert_df(ohlcv_history, len(ohlcv_history),
-                        ohlcv_history.loc[1, 'close'], ohlcv_history.loc[0, 'date'])
+                        ohlcv_history.loc[2, 'close'], ohlcv_history.loc[0, 'date'])
     assert log_has_re(r"Dataframe returned from strategy.*last date\.", caplog)
     # reset to avoid problems in other tests due to test leakage
     _STRATEGY.disable_dataframe_checks = False
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index e6c91a96e..bebbc1508 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -16,6 +16,7 @@ from freqtrade.configuration import (Configuration, check_exchange, remove_crede
 from freqtrade.configuration.config_validation import validate_config_schema
 from freqtrade.configuration.deprecated_settings import (check_conflicting_settings,
                                                          process_deprecated_setting,
+                                                         process_removed_setting,
                                                          process_temporary_deprecated_settings)
 from freqtrade.configuration.load_config import load_config_file, log_config_error_range
 from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL
@@ -879,6 +880,25 @@ def test_validate_whitelist(default_conf):
     validate_config_consistency(conf)
 
 
+@pytest.mark.parametrize('protconf,expected', [
+    ([], None),
+    ([{"method": "StoplossGuard", "lookback_period": 2000, "stop_duration_candles": 10}], None),
+    ([{"method": "StoplossGuard", "lookback_period_candles": 20, "stop_duration": 10}], None),
+    ([{"method": "StoplossGuard", "lookback_period_candles": 20, "lookback_period": 2000,
+       "stop_duration": 10}], r'Protections must specify either `lookback_period`.*'),
+    ([{"method": "StoplossGuard", "lookback_period": 20, "stop_duration": 10,
+       "stop_duration_candles": 10}], r'Protections must specify either `stop_duration`.*'),
+])
+def test_validate_protections(default_conf, protconf, expected):
+    conf = deepcopy(default_conf)
+    conf['protections'] = protconf
+    if expected:
+        with pytest.raises(OperationalException, match=expected):
+            validate_config_consistency(conf)
+    else:
+        validate_config_consistency(conf)
+
+
 def test_load_config_test_comments() -> None:
     """
     Load config with comments
@@ -1061,13 +1081,11 @@ def test_pairlist_resolving_fallback(mocker):
     assert config['datadir'] == Path.cwd() / "user_data/data/binance"
 
 
+@pytest.mark.skip(reason='Currently no deprecated / moved sections')
+# The below is kept as a sample for the future.
 @pytest.mark.parametrize("setting", [
         ("ask_strategy", "use_sell_signal", True,
          "experimental", "use_sell_signal", False),
-        ("ask_strategy", "sell_profit_only", False,
-         "experimental", "sell_profit_only", True),
-        ("ask_strategy", "ignore_roi_if_buy_signal", False,
-         "experimental", "ignore_roi_if_buy_signal", True),
     ])
 def test_process_temporary_deprecated_settings(mocker, default_conf, setting, caplog):
     patched_configuration_load_config_file(mocker, default_conf)
@@ -1097,7 +1115,27 @@ def test_process_temporary_deprecated_settings(mocker, default_conf, setting, ca
     assert default_conf[setting[0]][setting[1]] == setting[5]
 
 
-def test_process_deprecated_setting_edge(mocker, edge_conf, caplog):
+@pytest.mark.parametrize("setting", [
+        ("experimental", "use_sell_signal", False),
+        ("experimental", "sell_profit_only", True),
+        ("experimental", "ignore_roi_if_buy_signal", True),
+    ])
+def test_process_removed_settings(mocker, default_conf, setting):
+    patched_configuration_load_config_file(mocker, default_conf)
+
+    # Create sections for new and deprecated settings
+    # (they may not exist in the config)
+    default_conf[setting[0]] = {}
+    # Assign removed setting
+    default_conf[setting[0]][setting[1]] = setting[2]
+
+    # New and deprecated settings are conflicting ones
+    with pytest.raises(OperationalException,
+                       match=r'Setting .* has been moved'):
+        process_temporary_deprecated_settings(default_conf)
+
+
+def test_process_deprecated_setting_edge(mocker, edge_conf):
     patched_configuration_load_config_file(mocker, edge_conf)
     edge_conf.update({'edge': {
         'enabled': True,
@@ -1196,6 +1234,30 @@ def test_process_deprecated_setting(mocker, default_conf, caplog):
     assert default_conf['sectionA']['new_setting'] == 'valA'
 
 
+def test_process_removed_setting(mocker, default_conf, caplog):
+    patched_configuration_load_config_file(mocker, default_conf)
+
+    # Create sections for new and deprecated settings
+    # (they may not exist in the config)
+    default_conf['sectionA'] = {}
+    default_conf['sectionB'] = {}
+    # Assign new setting
+    default_conf['sectionB']['somesetting'] = 'valA'
+
+    # Only new setting exists (nothing should happen)
+    process_removed_setting(default_conf,
+                            'sectionA', 'somesetting',
+                            'sectionB', 'somesetting')
+    # Assign removed setting
+    default_conf['sectionA']['somesetting'] = 'valB'
+
+    with pytest.raises(OperationalException,
+                       match=r"Setting .* has been moved"):
+        process_removed_setting(default_conf,
+                                'sectionA', 'somesetting',
+                                'sectionB', 'somesetting')
+
+
 def test_process_deprecated_ticker_interval(mocker, default_conf, caplog):
     message = "DEPRECATED: Please use 'timeframe' instead of 'ticker_interval."
     config = deepcopy(default_conf)
diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py
index 1f5b3ecaa..12be5ae8b 100644
--- a/tests/test_freqtradebot.py
+++ b/tests/test_freqtradebot.py
@@ -15,7 +15,7 @@ from freqtrade.exceptions import (DependencyException, ExchangeError, Insufficie
                                   InvalidOrderException, OperationalException, PricingError,
                                   TemporaryError)
 from freqtrade.freqtradebot import FreqtradeBot
-from freqtrade.persistence import Order, Trade
+from freqtrade.persistence import Order, PairLocks, Trade
 from freqtrade.persistence.models import PairLock
 from freqtrade.rpc import RPCMessageType
 from freqtrade.state import RunMode, State
@@ -678,6 +678,32 @@ def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_o
     assert log_has("Active pair whitelist is empty.", caplog)
 
 
+@pytest.mark.usefixtures("init_persistence")
+def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, fee,
+                                         mocker, caplog) -> None:
+    patch_RPCManager(mocker)
+    patch_exchange(mocker)
+    mocker.patch.multiple(
+        'freqtrade.exchange.Exchange',
+        fetch_ticker=ticker,
+        buy=MagicMock(return_value={'id': limit_buy_order['id']}),
+        get_fee=fee,
+    )
+    freqtrade = FreqtradeBot(default_conf)
+    patch_get_signal(freqtrade)
+    n = freqtrade.enter_positions()
+    message = r"Global pairlock active until.* Not creating new trades."
+    n = freqtrade.enter_positions()
+    # 0 trades, but it's not because of pairlock.
+    assert n == 0
+    assert not log_has_re(message, caplog)
+
+    PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because')
+    n = freqtrade.enter_positions()
+    assert n == 0
+    assert log_has_re(message, caplog)
+
+
 def test_create_trade_no_signal(default_conf, fee, mocker) -> None:
     default_conf['dry_run'] = True
 
@@ -1074,6 +1100,12 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order
     mocker.patch('freqtrade.exchange.Exchange.buy', MagicMock(return_value=limit_buy_order))
     assert not freqtrade.execute_buy(pair, stake_amount)
 
+    # Fail to get price...
+    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_buy_rate', MagicMock(return_value=0.0))
+
+    with pytest.raises(PricingError, match="Could not determine buy price."):
+        freqtrade.execute_buy(pair, stake_amount)
+
 
 def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) -> None:
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
@@ -3257,7 +3289,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo
     caplog.clear()
     freqtrade.enter_positions()
 
-    assert log_has(f"Pair {trade.pair} is currently locked.", caplog)
+    assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog)
 
 
 def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open,
@@ -3556,7 +3588,7 @@ def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_b
     # Test if buy-signal is absent
     patch_get_signal(freqtrade, value=(False, True))
     assert freqtrade.handle_trade(trade) is True
-    assert trade.sell_reason == SellType.STOP_LOSS.value
+    assert trade.sell_reason == SellType.SELL_SIGNAL.value
 
 
 def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fee, caplog, mocker):
@@ -3712,6 +3744,48 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c
                    'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).',
                    caplog)
 
+    assert trade.fee_open == 0.001
+    assert trade.fee_close == 0.001
+    assert trade.fee_open_cost is not None
+    assert trade.fee_open_currency is not None
+    assert trade.fee_close_cost is None
+    assert trade.fee_close_currency is None
+
+
+def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee,
+                                mocker, markets):
+    # Different fee currency on both trades
+    mocker.patch('freqtrade.exchange.Exchange.get_trades_for_order', return_value=trades_for_order3)
+    amount = float(sum(x['amount'] for x in trades_for_order3))
+    default_conf['stake_currency'] = 'ETH'
+    trade = Trade(
+        pair='LTC/ETH',
+        amount=amount,
+        exchange='binance',
+        fee_open=fee.return_value,
+        fee_close=fee.return_value,
+        open_rate=0.245441,
+        open_order_id="123456"
+    )
+    # Fake markets entry to enable fee parsing
+    markets['BNB/ETH'] = markets['ETH/BTC']
+    freqtrade = get_patched_freqtradebot(mocker, default_conf)
+    mocker.patch('freqtrade.exchange.Exchange.markets', PropertyMock(return_value=markets))
+    mocker.patch('freqtrade.exchange.Exchange.fetch_ticker',
+                 return_value={'ask': 0.19, 'last': 0.2})
+
+    # Amount is reduced by "fee"
+    assert freqtrade.get_real_amount(trade, buy_order_fee) == amount - (amount * 0.0005)
+    assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, '
+                   'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).',
+                   caplog)
+    # Overall fee is average of both trade's fee
+    assert trade.fee_open == 0.001518575
+    assert trade.fee_open_cost is not None
+    assert trade.fee_open_currency is not None
+    assert trade.fee_close_cost is None
+    assert trade.fee_close_currency is None
+
 
 def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee,
                                    caplog, mocker):
@@ -4258,7 +4332,7 @@ def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee):
     freqtrade = get_patched_freqtradebot(mocker, default_conf)
 
     def patch_with_fee(order):
-        order.update({'fee': {'cost': 0.1, 'rate': 0.2,
+        order.update({'fee': {'cost': 0.1, 'rate': 0.01,
                       'currency': order['symbol'].split('/')[0]}})
         return order
 
diff --git a/tests/test_persistence.py b/tests/test_persistence.py
index 41b99b34f..7487b2ef5 100644
--- a/tests/test_persistence.py
+++ b/tests/test_persistence.py
@@ -177,10 +177,10 @@ def test_calc_open_close_trade_price(limit_buy_order, limit_sell_order, fee):
 
     trade.open_order_id = 'something'
     trade.update(limit_buy_order)
-    assert trade._calc_open_trade_price() == 0.0010024999999225068
+    assert trade._calc_open_trade_value() == 0.0010024999999225068
 
     trade.update(limit_sell_order)
-    assert trade.calc_close_trade_price() == 0.0010646656050132426
+    assert trade.calc_close_trade_value() == 0.0010646656050132426
 
     # Profit in BTC
     assert trade.calc_profit() == 0.00006217
@@ -233,7 +233,7 @@ def test_calc_close_trade_price_exception(limit_buy_order, fee):
 
     trade.open_order_id = 'something'
     trade.update(limit_buy_order)
-    assert trade.calc_close_trade_price() == 0.0
+    assert trade.calc_close_trade_value() == 0.0
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -277,7 +277,7 @@ def test_update_invalid_order(limit_buy_order):
 
 
 @pytest.mark.usefixtures("init_persistence")
-def test_calc_open_trade_price(limit_buy_order, fee):
+def test_calc_open_trade_value(limit_buy_order, fee):
     trade = Trade(
         pair='ETH/BTC',
         stake_amount=0.001,
@@ -291,10 +291,10 @@ def test_calc_open_trade_price(limit_buy_order, fee):
     trade.update(limit_buy_order)  # Buy @ 0.00001099
 
     # Get the open rate price with the standard fee rate
-    assert trade._calc_open_trade_price() == 0.0010024999999225068
+    assert trade._calc_open_trade_value() == 0.0010024999999225068
     trade.fee_open = 0.003
     # Get the open rate price with a custom fee rate
-    assert trade._calc_open_trade_price() == 0.001002999999922468
+    assert trade._calc_open_trade_value() == 0.001002999999922468
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -312,14 +312,14 @@ def test_calc_close_trade_price(limit_buy_order, limit_sell_order, fee):
     trade.update(limit_buy_order)  # Buy @ 0.00001099
 
     # Get the close rate price with a custom close rate and a regular fee rate
-    assert trade.calc_close_trade_price(rate=0.00001234) == 0.0011200318470471794
+    assert trade.calc_close_trade_value(rate=0.00001234) == 0.0011200318470471794
 
     # Get the close rate price with a custom close rate and a custom fee rate
-    assert trade.calc_close_trade_price(rate=0.00001234, fee=0.003) == 0.0011194704275749754
+    assert trade.calc_close_trade_value(rate=0.00001234, fee=0.003) == 0.0011194704275749754
 
     # Test when we apply a Sell order, and ask price with a custom fee rate
     trade.update(limit_sell_order)
-    assert trade.calc_close_trade_price(fee=0.005) == 0.0010619972701635854
+    assert trade.calc_close_trade_value(fee=0.005) == 0.0010619972701635854
 
 
 @pytest.mark.usefixtures("init_persistence")
@@ -499,7 +499,7 @@ def test_migrate_old(mocker, default_conf, fee):
     assert trade.max_rate == 0.0
     assert trade.stop_loss == 0.0
     assert trade.initial_stop_loss == 0.0
-    assert trade.open_trade_price == trade._calc_open_trade_price()
+    assert trade.open_trade_value == trade._calc_open_trade_value()
     assert trade.close_profit_abs is None
     assert trade.fee_open_cost is None
     assert trade.fee_open_currency is None
@@ -607,7 +607,7 @@ def test_migrate_new(mocker, default_conf, fee, caplog):
     assert log_has("trying trades_bak1", caplog)
     assert log_has("trying trades_bak2", caplog)
     assert log_has("Running database migration for trades - backup: trades_bak2", caplog)
-    assert trade.open_trade_price == trade._calc_open_trade_price()
+    assert trade.open_trade_value == trade._calc_open_trade_value()
     assert trade.close_profit_abs is None
 
     assert log_has("Moving open orders to Orders table.", caplog)
@@ -677,7 +677,7 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog):
     assert trade.max_rate == 0.0
     assert trade.stop_loss == 0.0
     assert trade.initial_stop_loss == 0.0
-    assert trade.open_trade_price == trade._calc_open_trade_price()
+    assert trade.open_trade_value == trade._calc_open_trade_value()
     assert log_has("trying trades_bak0", caplog)
     assert log_has("Running database migration for trades - backup: trades_bak0", caplog)
 
@@ -803,7 +803,7 @@ def test_to_json(default_conf, fee):
                       'close_timestamp': None,
                       'open_rate': 0.123,
                       'open_rate_requested': None,
-                      'open_trade_price': 15.1668225,
+                      'open_trade_value': 15.1668225,
                       'fee_close': 0.0025,
                       'fee_close_cost': None,
                       'fee_close_currency': None,
@@ -896,7 +896,7 @@ def test_to_json(default_conf, fee):
                       'min_rate': None,
                       'open_order_id': None,
                       'open_rate_requested': None,
-                      'open_trade_price': 12.33075,
+                      'open_trade_value': 12.33075,
                       'sell_reason': None,
                       'sell_order_status': None,
                       'strategy': None,
diff --git a/tests/test_plotting.py b/tests/test_plotting.py
index d3f97013d..42847ca50 100644
--- a/tests/test_plotting.py
+++ b/tests/test_plotting.py
@@ -13,7 +13,7 @@ from freqtrade.configuration import TimeRange
 from freqtrade.data import history
 from freqtrade.data.btanalysis import create_cum_profit, load_backtest_data
 from freqtrade.exceptions import OperationalException
-from freqtrade.plot.plotting import (add_indicators, add_profit, create_plotconfig,
+from freqtrade.plot.plotting import (add_areas, add_indicators, add_profit, create_plotconfig,
                                      generate_candlestick_graph, generate_plot_filename,
                                      generate_profit_graph, init_plotscript, load_and_plot_trades,
                                      plot_profit, plot_trades, store_plot_file)
@@ -96,6 +96,62 @@ def test_add_indicators(default_conf, testdatadir, caplog):
     assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog)
 
 
+def test_add_areas(default_conf, testdatadir, caplog):
+    pair = "UNITTEST/BTC"
+    timerange = TimeRange(None, 'line', 0, -1000)
+
+    data = history.load_pair_history(pair=pair, timeframe='1m',
+                                     datadir=testdatadir, timerange=timerange)
+    indicators = {"macd": {"color": "red",
+                           "fill_color": "black",
+                           "fill_to": "macdhist",
+                           "fill_label": "MACD Fill"}}
+
+    ind_no_label = {"macd": {"fill_color": "red",
+                             "fill_to": "macdhist"}}
+
+    ind_plain = {"macd": {"fill_to": "macdhist"}}
+    default_conf.update({'strategy': 'DefaultStrategy'})
+    strategy = StrategyResolver.load_strategy(default_conf)
+
+    # Generate buy/sell signals and indicators
+    data = strategy.analyze_ticker(data, {'pair': pair})
+    fig = generate_empty_figure()
+
+    # indicator mentioned in fill_to does not exist
+    fig1 = add_areas(fig, 1, data, {'ema10': {'fill_to': 'no_fill_indicator'}})
+    assert fig == fig1
+    assert log_has_re(r'fill_to: "no_fill_indicator" ignored\..*', caplog)
+
+    # indicator does not exist
+    fig2 = add_areas(fig, 1, data, {'no_indicator': {'fill_to': 'ema10'}})
+    assert fig == fig2
+    assert log_has_re(r'Indicator "no_indicator" ignored\..*', caplog)
+
+    # everythin given in plot config, row 3
+    fig3 = add_areas(fig, 3, data, indicators)
+    figure = fig3.layout.figure
+    fill_macd = find_trace_in_fig_data(figure.data, "MACD Fill")
+    assert isinstance(fill_macd, go.Scatter)
+    assert fill_macd.yaxis == "y3"
+    assert fill_macd.fillcolor == "black"
+
+    # label missing, row 1
+    fig4 = add_areas(fig, 1, data, ind_no_label)
+    figure = fig4.layout.figure
+    fill_macd = find_trace_in_fig_data(figure.data, "macd<>macdhist")
+    assert isinstance(fill_macd, go.Scatter)
+    assert fill_macd.yaxis == "y"
+    assert fill_macd.fillcolor == "red"
+
+    # fit_to only
+    fig5 = add_areas(fig, 1, data, ind_plain)
+    figure = fig5.layout.figure
+    fill_macd = find_trace_in_fig_data(figure.data, "macd<>macdhist")
+    assert isinstance(fill_macd, go.Scatter)
+    assert fill_macd.yaxis == "y"
+
+
 def test_plot_trades(testdatadir, caplog):
     fig1 = generate_empty_figure()
     # nothing happens when no trades are available