adelorenzo
/
stable
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into pr/gmatheu/4746

This commit is contained in:
Matthias 2021-05-23 16:03:11 +02:00
parent 77302ea178 971d5b2ecc
commit f760b4a789
166 changed files with 6354 additions and 2939 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.dockerignore
View File

@ -1,9 +1,8 @@
.git
.gitignore
Dockerfile
Dockerfile.armhf
.dockerignore
config.json*
*.sqlite
.coveragerc
.eggs
.github
@ -13,4 +12,13 @@ CONTRIBUTING.md
MANIFEST.in
README.md
freqtrade.service
freqtrade.egg-info
config.json*
*.sqlite
user_data
*.log
.vscode
.mypy_cache
.ipynb_checkpoints

3
.gitattributes vendored Normal file
View File

@ -0,0 +1,3 @@
*.py eol=lf
*.sh eol=lf
*.ps1 eol=crlf

6
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View File

@ -0,0 +1,6 @@
---
blank_issues_enabled: false
contact_links:
- name: Discord Server
url: https://discord.gg/MA9v74M
about: Ask a question or get community support from our Discord server

14
.github/workflows/ci.yml vendored
View File

@ -148,6 +148,7 @@ jobs:
- name: Installation - macOS
run: |
brew update
brew install hdf5 c-blosc
python -m pip install --upgrade pip
export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
@ -300,7 +301,7 @@ jobs:
runs-on: ubuntu-20.04
steps:
- name: Cleanup previous runs on this branch
uses: rokroskar/workflow-run-cleanup-action@v0.2.2
uses: rokroskar/workflow-run-cleanup-action@v0.3.3
if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'"
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
@ -310,9 +311,18 @@ jobs:
needs: [ build_linux, build_macos, build_windows, docs_check ]
runs-on: ubuntu-20.04
steps:
- name: Check user permission
id: check
uses: scherermichael-oss/action-has-permission@1.0.6
with:
required-permission: write
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Slack Notification
uses: lazy-actions/slatify@v3.0.0
if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
if: always() && steps.check.outputs.has-permission && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
with:
type: ${{ job.status }}
job_name: '*Freqtrade CI*'

28
Dockerfile
View File

@ -1,14 +1,23 @@
FROM python:3.9.2-slim-buster as base
FROM python:3.9.5-slim-buster as base
# Setup env
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONFAULTHANDLER 1
ENV PATH=/root/.local/bin:$PATH
ENV PATH=/home/ftuser/.local/bin:$PATH
ENV FT_APP_ENV="docker"
# Prepare environment
RUN mkdir /freqtrade
RUN mkdir /freqtrade \
&& apt update \
&& apt install -y sudo \
&& apt-get clean \
&& useradd -u 1000 -G sudo -U -m ftuser \
&& chown ftuser:ftuser /freqtrade \
# Allow sudoers
&& echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
WORKDIR /freqtrade
# Install dependencies
@ -24,7 +33,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
ENV LD_LIBRARY_PATH /usr/local/lib
# Install dependencies
COPY requirements.txt requirements-hyperopt.txt /freqtrade/
COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
USER ftuser
RUN pip install --user --no-cache-dir numpy \
&& pip install --user --no-cache-dir -r requirements-hyperopt.txt
@ -33,13 +43,13 @@ FROM base as runtime-image
COPY --from=python-deps /usr/local/lib /usr/local/lib
ENV LD_LIBRARY_PATH /usr/local/lib
COPY --from=python-deps /root/.local /root/.local
COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local
USER ftuser
# Install and execute
COPY . /freqtrade/
RUN pip install -e . --no-cache-dir \
COPY --chown=ftuser:ftuser . /freqtrade/
RUN pip install -e . --user --no-cache-dir \
&& mkdir /freqtrade/user_data/ \
&& freqtrade install-ui

33
Dockerfile.armhf
View File

@ -1,19 +1,24 @@
FROM --platform=linux/arm/v7 python:3.7.9-slim-buster as base
FROM --platform=linux/arm/v7 python:3.7.10-slim-buster as base
# Setup env
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONFAULTHANDLER 1
ENV PATH=/root/.local/bin:$PATH
ENV PATH=/home/ftuser/.local/bin:$PATH
ENV FT_APP_ENV="docker"
# Prepare environment
RUN mkdir /freqtrade
WORKDIR /freqtrade
RUN mkdir /freqtrade \
&& apt-get update \
&& apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \
&& apt-get clean \
&& useradd -u 1000 -G sudo -U -m ftuser \
&& chown ftuser:ftuser /freqtrade \
# Allow sudoers
&& echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
RUN apt-get update \
&& apt-get -y install libatlas3-base curl sqlite3 \
&& apt-get clean
WORKDIR /freqtrade
# Install dependencies
FROM base as python-deps
@ -28,7 +33,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
ENV LD_LIBRARY_PATH /usr/local/lib
# Install dependencies
COPY requirements.txt /freqtrade/
COPY --chown=ftuser:ftuser requirements.txt /freqtrade/
USER ftuser
RUN pip install --user --no-cache-dir numpy \
&& pip install --user --no-cache-dir -r requirements.txt
@ -37,13 +43,14 @@ FROM base as runtime-image
COPY --from=python-deps /usr/local/lib /usr/local/lib
ENV LD_LIBRARY_PATH /usr/local/lib
COPY --from=python-deps /root/.local /root/.local
COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local
USER ftuser
# Install and execute
COPY . /freqtrade/
RUN apt-get install -y libhdf5-serial-dev \
&& apt-get clean \
&& pip install -e . --no-cache-dir \
COPY --chown=ftuser:ftuser . /freqtrade/
RUN pip install -e . --user --no-cache-dir \
&& mkdir /freqtrade/user_data/ \
&& freqtrade install-ui
ENTRYPOINT ["freqtrade"]

6
README.md
View File

@ -1,4 +1,4 @@
# Freqtrade
# ![freqtrade](https://raw.githubusercontent.com/freqtrade/freqtrade/develop/docs/assets/freqtrade_poweredby.svg)
[![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/)
[![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop)
@ -154,7 +154,7 @@ You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/
If you discover a bug in the bot, please
[search our issue tracker](https://github.com/freqtrade/freqtrade/issues?q=is%3Aissue)
first. If it hasn't been reported, please
[create a new issue](https://github.com/freqtrade/freqtrade/issues/new) and
[create a new issue](https://github.com/freqtrade/freqtrade/issues/new/choose) and
ensure you follow the template guide so that our team can assist you as
quickly as possible.
@ -163,7 +163,7 @@ quickly as possible.
Have you a great idea to improve the bot you want to share? Please,
first search if this feature was not [already discussed](https://github.com/freqtrade/freqtrade/labels/enhancement).
If it hasn't been requested, please
[create a new request](https://github.com/freqtrade/freqtrade/issues/new)
[create a new request](https://github.com/freqtrade/freqtrade/issues/new/choose)
and ensure you follow the template guide so that it does not get lost
in the bug reports.

5
build_helpers/install_ta-lib.sh
View File

@ -8,10 +8,13 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then
tar zxvf ta-lib-0.4.0-src.tar.gz
cd ta-lib \
&& sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \
&& curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess \
&& curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \
&& ./configure --prefix=${INSTALL_LOC}/ \
&& make \
&& make -j$(nproc) \
&& which sudo && sudo make install || make install \
&& cd ..
else
echo "TA-lib already installed, skipping installation"
fi
# && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \

99
config_ftx.json.example Normal file
View File

@ -0,0 +1,99 @@
{
"max_open_trades": 3,
"stake_currency": "USD",
"stake_amount": 50,
"tradable_balance_ratio": 0.99,
"fiat_display_currency": "USD",
"timeframe": "5m",
"dry_run": true,
"cancel_open_orders_on_exit": false,
"unfilledtimeout": {
"buy": 10,
"sell": 30
},
"bid_strategy": {
"ask_last_balance": 0.0,
"use_order_book": false,
"order_book_top": 1,
"check_depth_of_market": {
"enabled": false,
"bids_to_ask_delta": 1
}
},
"ask_strategy": {
"use_order_book": false,
"order_book_min": 1,
"order_book_max": 1,
"use_sell_signal": true,
"sell_profit_only": false,
"ignore_roi_if_buy_signal": false
},
"exchange": {
"name": "ftx",
"key": "your_exchange_key",
"secret": "your_exchange_secret",
"ccxt_config": {"enableRateLimit": true},
"ccxt_async_config": {
"enableRateLimit": true,
"rateLimit": 50
},
"pair_whitelist": [
"BTC/USD",
"ETH/USD",
"BNB/USD",
"USDT/USD",
"LTC/USD",
"SRM/USD",
"SXP/USD",
"XRP/USD",
"DOGE/USD",
"1INCH/USD",
"CHZ/USD",
"MATIC/USD",
"LINK/USD",
"OXY/USD",
"SUSHI/USD"
],
"pair_blacklist": [
"FTT/USD"
]
},
"pairlists": [
{"method": "StaticPairList"}
],
"edge": {
"enabled": false,
"process_throttle_secs": 3600,
"calculate_since_number_of_days": 7,
"allowed_risk": 0.01,
"stoploss_range_min": -0.01,
"stoploss_range_max": -0.1,
"stoploss_range_step": -0.01,
"minimum_winrate": 0.60,
"minimum_expectancy": 0.20,
"min_trade_number": 10,
"max_trade_duration_minute": 1440,
"remove_pumps": false
},
"telegram": {
"enabled": false,
"token": "your_telegram_token",
"chat_id": "your_telegram_chat_id"
},
"api_server": {
"enabled": false,
"listen_ip_address": "127.0.0.1",
"listen_port": 8080,
"verbosity": "error",
"jwt_secret_key": "somethingrandom",
"CORS_origins": [],
"username": "freqtrader",
"password": "SuperSecurePassword"
},
"bot_name": "freqtrade",
"initial_state": "running",
"forcebuy_enable": false,
"internals": {
"process_throttle_secs": 5
}
}

5
config_full.json.example
View File

@ -23,7 +23,8 @@
"stoploss": -0.10,
"unfilledtimeout": {
"buy": 10,
"sell": 30
"sell": 30,
"unit": "minutes"
},
"bid_strategy": {
"price_side": "bid",
@ -163,7 +164,9 @@
"warning": "on",
"startup": "on",
"buy": "on",
"buy_fill": "on",
"sell": "on",
"sell_fill": "on",
"buy_cancel": "on",
"sell_cancel": "on"
}

2
docker-compose.yml
View File

@ -9,7 +9,7 @@ services:
# Build step - only needed when additional dependencies are needed
# build:
# context: .
# dockerfile: "./docker/Dockerfile.technical"
# dockerfile: "./docker/Dockerfile.custom"
restart: unless-stopped
container_name: freqtrade
volumes:

58
docker/Dockerfile.aarch64 Normal file
View File

@ -0,0 +1,58 @@
FROM --platform=linux/arm64/v8 python:3.9.4-slim-buster as base
# Setup env
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
ENV PYTHONDONTWRITEBYTECODE 1
ENV PYTHONFAULTHANDLER 1
ENV PATH=/home/ftuser/.local/bin:$PATH
ENV FT_APP_ENV="docker"
# Prepare environment
RUN mkdir /freqtrade \
&& apt-get update \
&& apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \
&& apt-get clean \
&& useradd -u 1000 -G sudo -U -m ftuser \
&& chown ftuser:ftuser /freqtrade \
# Allow sudoers
&& echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
WORKDIR /freqtrade
# Install dependencies
FROM base as python-deps
RUN apt-get update \
&& apt-get -y install curl build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \
&& apt-get clean \
&& pip install --upgrade pip
# Install TA-lib
COPY build_helpers/* /tmp/
RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
ENV LD_LIBRARY_PATH /usr/local/lib
# Install dependencies
COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
USER ftuser
RUN pip install --user --no-cache-dir numpy \
&& pip install --user --no-cache-dir -r requirements-hyperopt.txt
# Copy dependencies to runtime-image
FROM base as runtime-image
COPY --from=python-deps /usr/local/lib /usr/local/lib
ENV LD_LIBRARY_PATH /usr/local/lib
COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local
USER ftuser
# Install and execute
COPY --chown=ftuser:ftuser . /freqtrade/
RUN pip install -e . --user --no-cache-dir \
&& mkdir /freqtrade/user_data/ \
&& freqtrade install-ui
ENTRYPOINT ["freqtrade"]
# Default to trade mode
CMD [ "trade" ]

10
docker/Dockerfile.custom Normal file
View File

@ -0,0 +1,10 @@
FROM freqtradeorg/freqtrade:develop
# Switch user to root if you must install something from apt
# Don't forget to switch the user back below!
# USER root
# The below dependency - pyti - serves as an example. Please use whatever you need!
RUN pip install --user pyti
# USER ftuser

4
docker/Dockerfile.develop
View File

@ -3,8 +3,8 @@ FROM freqtradeorg/freqtrade:develop
# Install dependencies
COPY requirements-dev.txt /freqtrade/
RUN pip install numpy --no-cache-dir \
&& pip install -r requirements-dev.txt --no-cache-dir
RUN pip install numpy --user --no-cache-dir \
&& pip install -r requirements-dev.txt --user --no-cache-dir
# Empty the ENTRYPOINT to allow all commands
ENTRYPOINT []

2
docker/Dockerfile.jupyter
View File

@ -1,7 +1,7 @@
FROM freqtradeorg/freqtrade:develop_plot
RUN pip install jupyterlab --no-cache-dir
RUN pip install jupyterlab --user --no-cache-dir
# Empty the ENTRYPOINT to allow all commands
ENTRYPOINT []

2
docker/Dockerfile.plot
View File

@ -4,4 +4,4 @@ FROM freqtradeorg/freqtrade:${sourceimage}
# Install dependencies
COPY requirements-plot.txt /freqtrade/
RUN pip install -r requirements-plot.txt --no-cache-dir
RUN pip install -r requirements-plot.txt --user --no-cache-dir

6
docker/Dockerfile.technical
View File

@ -1,6 +0,0 @@
FROM freqtradeorg/freqtrade:develop
RUN apt-get update \
&& apt-get -y install git \
&& apt-get clean \
&& pip install git+https://github.com/freqtrade/technical

385
docs/advanced-hyperopt.md
View File

@ -4,79 +4,6 @@ This page explains some advanced Hyperopt topics that may require higher
coding skills and Python knowledge than creation of an ordinal hyperoptimization
class.
## Derived hyperopt classes
Custom hyperopt classes can be derived in the same way [it can be done for strategies](strategy-customization.md#derived-strategies).
Applying to hyperoptimization, as an example, you may override how dimensions are defined in your optimization hyperspace:
```python
class MyAwesomeHyperOpt(IHyperOpt):
...
# Uses default stoploss dimension
class MyAwesomeHyperOpt2(MyAwesomeHyperOpt):
@staticmethod
def stoploss_space() -> List[Dimension]:
# Override boundaries for stoploss
return [
Real(-0.33, -0.01, name='stoploss'),
]
```
and then quickly switch between hyperopt classes, running optimization process with hyperopt class you need in each particular case:
```
$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy ...
or
$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt2 --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy ...
```
## Sharing methods with your strategy
Hyperopt classes provide access to the Strategy via the `strategy` class attribute.
This can be a great way to reduce code duplication if used correctly, but will also complicate usage for inexperienced users.
``` python
from pandas import DataFrame
from freqtrade.strategy.interface import IStrategy
import freqtrade.vendor.qtpylib.indicators as qtpylib
class MyAwesomeStrategy(IStrategy):
buy_params = {
'rsi-value': 30,
'adx-value': 35,
}
def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
return self.buy_strategy_generator(self.buy_params, dataframe, metadata)
@staticmethod
def buy_strategy_generator(params, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe.loc[
(
qtpylib.crossed_above(dataframe['rsi'], params['rsi-value']) &
dataframe['adx'] > params['adx-value']) &
dataframe['volume'] > 0
)
, 'buy'] = 1
return dataframe
class MyAwesomeHyperOpt(IHyperOpt):
...
@staticmethod
def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
"""
Define the buy strategy parameters to be used by Hyperopt.
"""
def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
# Call strategy's buy strategy generator
return self.StrategyClass.buy_strategy_generator(params, dataframe, metadata)
return populate_buy_trend
```
## Creating and using a custom loss function
To use a custom loss function class, make sure that the function `hyperopt_loss_function` is defined in your custom hyperopt loss class.
@ -142,3 +69,315 @@ This function needs to return a floating point number (`float`). Smaller numbers
!!! Note
Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface later.
## Overriding pre-defined spaces
To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_space`, `trailing_space`), define a nested class called Hyperopt and define the required spaces as follows:
```python
class MyAwesomeStrategy(IStrategy):
class HyperOpt:
# Define a custom stoploss space.
def stoploss_space(self):
return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')]
```
## Space options
For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types:
* `Categorical` - Pick from a list of categories (e.g. `Categorical(['a', 'b', 'c'], name="cat")`)
* `Integer` - Pick from a range of whole numbers (e.g. `Integer(1, 10, name='rsi')`)
* `SKDecimal` - Pick from a range of decimal numbers with limited precision (e.g. `SKDecimal(0.1, 0.5, decimals=3, name='adx')`). *Available only with freqtrade*.
* `Real` - Pick from a range of decimal numbers with full precision (e.g. `Real(0.1, 0.5, name='adx')`
You can import all of these from `freqtrade.optimize.space`, although `Categorical`, `Integer` and `Real` are only aliases for their corresponding scikit-optimize Spaces. `SKDecimal` is provided by freqtrade for faster optimizations.
``` python
from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real # noqa
```
!!! Hint "SKDecimal vs. Real"
We recommend to use `SKDecimal` instead of the `Real` space in almost all cases. While the Real space provides full accuracy (up to ~16 decimal places) - this precision is rarely needed, and leads to unnecessary long hyperopt times.
Assuming the definition of a rather small space (`SKDecimal(0.10, 0.15, decimals=2, name='xxx')`) - SKDecimal will have 5 possibilities (`[0.10, 0.11, 0.12, 0.13, 0.14, 0.15]`).
A corresponding real space `Real(0.10, 0.15 name='xxx')` on the other hand has an almost unlimited number of possibilities (`[0.10, 0.010000000001, 0.010000000002, ... 0.014999999999, 0.01500000000]`).
---
## Legacy Hyperopt
This Section explains the configuration of an explicit Hyperopt file (separate to the strategy).
!!! Warning "Deprecated / legacy mode"
Since the 2021.4 release you no longer have to write a separate hyperopt class, but all strategies can be hyperopted.
Please read the [main hyperopt page](hyperopt.md) for more details.
### Prepare hyperopt file
Configuring an explicit hyperopt file is similar to writing your own strategy, and many tasks will be similar.
!!! Tip "About this page"
For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class.
#### Create a Custom Hyperopt File
The simplest way to get started is to use the following command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`.
Let assume you want a hyperopt file `AwesomeHyperopt.py`:
``` bash
freqtrade new-hyperopt --hyperopt AwesomeHyperopt
```
#### Legacy Hyperopt checklist
Checklist on all tasks / possibilities in hyperopt
Depending on the space you want to optimize, only some of the below are required:
* fill `buy_strategy_generator` - for buy signal optimization
* fill `indicator_space` - for buy signal optimization
* fill `sell_strategy_generator` - for sell signal optimization
* fill `sell_indicator_space` - for sell signal optimization
!!! Note
`populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work.
Optional in hyperopt - can also be loaded from a strategy (recommended):
* `populate_indicators` - fallback to create indicators
* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy
* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy
!!! Note
You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods.
Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead.
Rarely you may also need to override:
* `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default)
* `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps)
* `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default)
* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default)
#### Defining a buy signal optimization
Let's say you are curious: should you use MACD crossings or lower Bollinger
Bands to trigger your buys. And you also wonder should you use RSI or ADX to
help with those buy decisions. If you decide to use RSI or ADX, which values
should I use for them? So let's use hyperparameter optimization to solve this
mystery.
We will start by defining a search space:
```python
def indicator_space() -> List[Dimension]:
"""
Define your Hyperopt space for searching strategy parameters
"""
return [
Integer(20, 40, name='adx-value'),
Integer(20, 40, name='rsi-value'),
Categorical([True, False], name='adx-enabled'),
Categorical([True, False], name='rsi-enabled'),
Categorical(['bb_lower', 'macd_cross_signal'], name='trigger')
]
```
Above definition says: I have five parameters I want you to randomly combine
to find the best combination. Two of them are integer values (`adx-value` and `rsi-value`) and I want you test in the range of values 20 to 40.
Then we have three category variables. First two are either `True` or `False`.
We use these to either enable or disable the ADX and RSI guards.
The last one we call `trigger` and use it to decide which buy trigger we want to use.
So let's write the buy strategy generator using these values:
```python
@staticmethod
def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
"""
Define the buy strategy parameters to be used by Hyperopt.
"""
def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
conditions = []
# GUARDS AND TRENDS
if 'adx-enabled' in params and params['adx-enabled']:
conditions.append(dataframe['adx'] > params['adx-value'])
if 'rsi-enabled' in params and params['rsi-enabled']:
conditions.append(dataframe['rsi'] < params['rsi-value'])
# TRIGGERS
if 'trigger' in params:
if params['trigger'] == 'bb_lower':
conditions.append(dataframe['close'] < dataframe['bb_lowerband'])
if params['trigger'] == 'macd_cross_signal':
conditions.append(qtpylib.crossed_above(
dataframe['macd'], dataframe['macdsignal']
))
# Check that volume is not 0
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
'buy'] = 1
return dataframe
return populate_buy_trend
```
Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations.
It will use the given historical data and make buys based on the buy signals generated with the above function.
Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)).
!!! Note
The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators.
When you want to test an indicator that isn't used by the bot currently, remember to
add it to the `populate_indicators()` method in your strategy or hyperopt file.
#### Sell optimization
Similar to the buy-signal above, sell-signals can also be optimized.
Place the corresponding settings into the following methods
* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing.
* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters.
The configuration and rules are the same than for buy signals.
To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`.
### Execute Hyperopt
Once you have updated your hyperopt configuration you can run it.
Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. More time usually results in better results.
We strongly recommend to use `screen` or `tmux` to prevent any connection loss.
```bash
freqtrade hyperopt --config config.json --hyperopt <hyperoptname> --hyperopt-loss <hyperoptlossname> --strategy <strategyname> -e 500 --spaces all
```
Use `<hyperoptname>` as the name of the custom hyperopt used.
The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs.
Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results.
The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below.
!!! Note
Hyperopt will store hyperopt results with the timestamp of the hyperopt start time.
Reading commands (`hyperopt-list`, `hyperopt-show`) can use `--hyperopt-filename <filename>` to read and display older hyperopt results.
You can find a list of filenames with `ls -l user_data/hyperopt_results/`.
#### Running Hyperopt using methods from a strategy
Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided.
```bash
freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy
```
### Understand the Hyperopt Result
Once Hyperopt is completed you can use the result to create a new strategy.
Given the following result from hyperopt:
```
Best result:
44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
Buy hyperspace params:
{ 'adx-value': 44,
'rsi-value': 29,
'adx-enabled': False,
'rsi-enabled': True,
'trigger': 'bb_lower'}
```
You should understand this result like:
* The buy trigger that worked best was `bb_lower`.
* You should not use ADX because `adx-enabled: False`)
* You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`)
You have to look inside your strategy file into `buy_strategy_generator()`
method, what those values match to.
So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block:
```python
(dataframe['rsi'] < 29.0)
```
Translating your whole hyperopt result as the new buy-signal would then look like:
```python
def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
dataframe.loc[
(
(dataframe['rsi'] < 29.0) & # rsi-value
dataframe['close'] < dataframe['bb_lowerband'] # trigger
),
'buy'] = 1
return dataframe
```
### Validate backtesting results
Once the optimized parameters and conditions have been implemented into your strategy, you should backtest the strategy to make sure everything is working as expected.
To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting.
Should results don't match, please double-check to make sure you transferred all conditions correctly.
Pay special care to the stoploss (and trailing stoploss) parameters, as these are often set in configuration files, which override changes to the strategy.
You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss` or `trailing_stop`).
### Sharing methods with your strategy
Hyperopt classes provide access to the Strategy via the `strategy` class attribute.
This can be a great way to reduce code duplication if used correctly, but will also complicate usage for inexperienced users.
``` python
from pandas import DataFrame
from freqtrade.strategy.interface import IStrategy
import freqtrade.vendor.qtpylib.indicators as qtpylib
class MyAwesomeStrategy(IStrategy):
buy_params = {
'rsi-value': 30,
'adx-value': 35,
}
def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
return self.buy_strategy_generator(self.buy_params, dataframe, metadata)
@staticmethod
def buy_strategy_generator(params, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe.loc[
(
qtpylib.crossed_above(dataframe['rsi'], params['rsi-value']) &
dataframe['adx'] > params['adx-value']) &
dataframe['volume'] > 0
)
, 'buy'] = 1
return dataframe
class MyAwesomeHyperOpt(IHyperOpt):
...
@staticmethod
def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
"""
Define the buy strategy parameters to be used by Hyperopt.
"""
def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
# Call strategy's buy strategy generator
return self.StrategyClass.buy_strategy_generator(params, dataframe, metadata)
return populate_buy_trend
```

3
docs/assets/ccxt-logo.svg Normal file
View File

@ -0,0 +1,3 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid meet" viewBox="0 0 90 90" width="100" height="100"><defs><path d="M0 90L0 0L90 0L90 90L0 90ZM50 60L60 60L60 80L70 80L70 60L80 60L80 50L50 50L50 60ZM30 80L40 80L40 70L30 70L30 80ZM30 60L20 60L20 70L10 70L10 80L20 80L20 70L30 70L30 60L40 60L40 50L30 50L30 60ZM10 60L20 60L20 50L10 50L10 60ZM10 40L40 40L40 30L20 30L20 20L40 20L40 10L10 10L10 40ZM50 40L80 40L80 30L60 30L60 20L80 20L80 10L50 10L50 40Z" id="c6g67PWSoP"></path></defs><g><g><g><use xlink:href="#c6g67PWSoP" opacity="1" fill="#000000" fill-opacity="1"></use></g></g></g></svg>
Side by Side

After

Width:  |  Height:  |  Size: 818 B

44
docs/assets/freqtrade_poweredby.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 18 KiB

84
docs/backtesting.md
View File

@ -15,7 +15,8 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[--data-format-ohlcv {json,jsongz,hdf5}]
[--max-open-trades INT]
[--stake-amount STAKE_AMOUNT] [--fee FLOAT]
[--eps] [--dmmp] [--enable-protections]
[-p PAIRS [PAIRS ...]] [--eps] [--dmmp]
[--enable-protections]
[--dry-run-wallet DRY_RUN_WALLET]
[--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
[--export EXPORT] [--export-filename PATH]
@ -23,8 +24,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH]
optional arguments:
-h, --help show this help message and exit
-i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
`1d`).
Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
--timerange TIMERANGE
Specify what timerange of data to use.
--data-format-ohlcv {json,jsongz,hdf5}
@ -38,6 +38,9 @@ optional arguments:
setting.
--fee FLOAT Specify fee ratio. Will be applied twice (on trade
entry and exit).
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
Limit command to these pairs. Pairs are space-
separated.
--eps, --enable-position-stacking
Allow buying the same pair multiple times (position
stacking).
@ -234,29 +237,29 @@ The most important in the backtesting is to understand the result.
A backtesting result will look like that:
```
========================================================= BACKTESTING REPORT ========================================================
| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |
|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:|
| ADA/BTC | 35 | -0.11 | -3.88 | -0.00019428 | -1.94 | 4:35:00 | 14 | 0 | 21 |
| ARK/BTC | 11 | -0.41 | -4.52 | -0.00022647 | -2.26 | 2:03:00 | 3 | 0 | 8 |
| BTS/BTC | 32 | 0.31 | 9.78 | 0.00048938 | 4.89 | 5:05:00 | 18 | 0 | 14 |
| DASH/BTC | 13 | -0.08 | -1.07 | -0.00005343 | -0.53 | 4:39:00 | 6 | 0 | 7 |
| ENG/BTC | 18 | 1.36 | 24.54 | 0.00122807 | 12.27 | 2:50:00 | 8 | 0 | 10 |
| EOS/BTC | 36 | 0.08 | 3.06 | 0.00015304 | 1.53 | 3:34:00 | 16 | 0 | 20 |
| ETC/BTC | 26 | 0.37 | 9.51 | 0.00047576 | 4.75 | 6:14:00 | 11 | 0 | 15 |
| ETH/BTC | 33 | 0.30 | 9.96 | 0.00049856 | 4.98 | 7:31:00 | 16 | 0 | 17 |
| IOTA/BTC | 32 | 0.03 | 1.09 | 0.00005444 | 0.54 | 3:12:00 | 14 | 0 | 18 |
| LSK/BTC | 15 | 1.75 | 26.26 | 0.00131413 | 13.13 | 2:58:00 | 6 | 0 | 9 |
| LTC/BTC | 32 | -0.04 | -1.38 | -0.00006886 | -0.69 | 4:49:00 | 11 | 0 | 21 |
| NANO/BTC | 17 | 1.26 | 21.39 | 0.00107058 | 10.70 | 1:55:00 | 10 | 0 | 7 |
| NEO/BTC | 23 | 0.82 | 18.97 | 0.00094936 | 9.48 | 2:59:00 | 10 | 0 | 13 |
| REQ/BTC | 9 | 1.17 | 10.54 | 0.00052734 | 5.27 | 3:47:00 | 4 | 0 | 5 |
| XLM/BTC | 16 | 1.22 | 19.54 | 0.00097800 | 9.77 | 3:15:00 | 7 | 0 | 9 |
| XMR/BTC | 23 | -0.18 | -4.13 | -0.00020696 | -2.07 | 5:30:00 | 12 | 0 | 11 |
| XRP/BTC | 35 | 0.66 | 22.96 | 0.00114897 | 11.48 | 3:49:00 | 12 | 0 | 23 |
| ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 | 0 | 15 |
| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 |
========================================================= SELL REASON STATS =========================================================
========================================================= BACKTESTING REPORT ==========================================================
| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins Draws Loss Win% |
|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:-------------|-------------------------:|
| ADA/BTC | 35 | -0.11 | -3.88 | -0.00019428 | -1.94 | 4:35:00 | 14 0 21 40.0 |
| ARK/BTC | 11 | -0.41 | -4.52 | -0.00022647 | -2.26 | 2:03:00 | 3 0 8 27.3 |
| BTS/BTC | 32 | 0.31 | 9.78 | 0.00048938 | 4.89 | 5:05:00 | 18 0 14 56.2 |
| DASH/BTC | 13 | -0.08 | -1.07 | -0.00005343 | -0.53 | 4:39:00 | 6 0 7 46.2 |
| ENG/BTC | 18 | 1.36 | 24.54 | 0.00122807 | 12.27 | 2:50:00 | 8 0 10 44.4 |
| EOS/BTC | 36 | 0.08 | 3.06 | 0.00015304 | 1.53 | 3:34:00 | 16 0 20 44.4 |
| ETC/BTC | 26 | 0.37 | 9.51 | 0.00047576 | 4.75 | 6:14:00 | 11 0 15 42.3 |
| ETH/BTC | 33 | 0.30 | 9.96 | 0.00049856 | 4.98 | 7:31:00 | 16 0 17 48.5 |
| IOTA/BTC | 32 | 0.03 | 1.09 | 0.00005444 | 0.54 | 3:12:00 | 14 0 18 43.8 |
| LSK/BTC | 15 | 1.75 | 26.26 | 0.00131413 | 13.13 | 2:58:00 | 6 0 9 40.0 |
| LTC/BTC | 32 | -0.04 | -1.38 | -0.00006886 | -0.69 | 4:49:00 | 11 0 21 34.4 |
| NANO/BTC | 17 | 1.26 | 21.39 | 0.00107058 | 10.70 | 1:55:00 | 10 0 7 58.5 |
| NEO/BTC | 23 | 0.82 | 18.97 | 0.00094936 | 9.48 | 2:59:00 | 10 0 13 43.5 |
| REQ/BTC | 9 | 1.17 | 10.54 | 0.00052734 | 5.27 | 3:47:00 | 4 0 5 44.4 |
| XLM/BTC | 16 | 1.22 | 19.54 | 0.00097800 | 9.77 | 3:15:00 | 7 0 9 43.8 |
| XMR/BTC | 23 | -0.18 | -4.13 | -0.00020696 | -2.07 | 5:30:00 | 12 0 11 52.2 |
| XRP/BTC | 35 | 0.66 | 22.96 | 0.00114897 | 11.48 | 3:49:00 | 12 0 23 34.3 |
| ZEC/BTC | 22 | -0.46 | -10.18 | -0.00050971 | -5.09 | 2:22:00 | 7 0 15 31.8 |
| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 0 243 43.4 |
========================================================= SELL REASON STATS ==========================================================
| Sell Reason | Sells | Wins | Draws | Losses |
|:-------------------|--------:|------:|-------:|--------:|
| trailing_stop_loss | 205 | 150 | 0 | 55 |
@ -264,11 +267,11 @@ A backtesting result will look like that:
| sell_signal | 56 | 36 | 0 | 20 |
| force_sell | 2 | 0 | 0 | 2 |
====================================================== LEFT OPEN TRADES REPORT ======================================================
| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |
|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|--------:|
| ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 | 0 | 0 |
| LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 | 0 | 0 |
| TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 | 0 | 0 |
| Pair | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Win Draw Loss Win% |
|:---------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|--------------------:|
| ADA/BTC | 1 | 0.89 | 0.89 | 0.00004434 | 0.44 | 6:00:00 | 1 0 0 100 |
| LTC/BTC | 1 | 0.68 | 0.68 | 0.00003421 | 0.34 | 2:00:00 | 1 0 0 100 |
| TOTAL | 2 | 0.78 | 1.57 | 0.00007855 | 0.78 | 4:00:00 | 2 0 0 100 |
=============== SUMMARY METRICS ===============
| Metric | Value |
|-----------------------+---------------------|
@ -294,6 +297,8 @@ A backtesting result will look like that:
| Days win/draw/lose | 12 / 82 / 25 |
| Avg. Duration Winners | 4:23:00 |
| Avg. Duration Loser | 6:55:00 |
| Zero Duration Trades | 4.6% (20) |
| Rejected Buy signals | 3089 |
| | |
| Min balance | 0.00945123 BTC |
| Max balance | 0.01846651 BTC |
@ -315,7 +320,7 @@ The last line will give you the overall performance of your strategy,
here:
```
| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 243 |
| TOTAL | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 0 243 43.4 |
```
The bot has made `429` trades for an average duration of `4:12:00`, with a performance of `76.20%` (profit), that means it has
@ -381,6 +386,8 @@ It contains some useful key metrics about performance of your strategy on backte
| Days win/draw/lose | 12 / 82 / 25 |
| Avg. Duration Winners | 4:23:00 |
| Avg. Duration Loser | 6:55:00 |
| Zero Duration Trades | 4.6% (20) |
| Rejected Buy signals | 3089 |
| | |
| Min balance | 0.00945123 BTC |
| Max balance | 0.01846651 BTC |
@ -410,6 +417,8 @@ It contains some useful key metrics about performance of your strategy on backte
- `Best day` / `Worst day`: Best and worst day based on daily profit.
- `Days win/draw/lose`: Winning / Losing days (draws are usually days without closed trade).
- `Avg. Duration Winners` / `Avg. Duration Loser`: Average durations for winning and losing trades.
- `Zero Duration Trades`: A number of trades that completed within same candle as they opened and had `trailing_stop_loss` sell reason. A significant amount of such trades may indicate that strategy is exploiting trailing stoploss behavior in backtesting and produces unrealistic results.
- `Rejected Buy signals`: Buy signals that could not be acted upon due to max_open_trades being reached.
- `Min balance` / `Max balance`: Lowest and Highest Wallet balance during the backtest period.
- `Drawdown`: Maximum drawdown experienced. For example, the value of 50% means that from highest to subsequent lowest point, a 50% drop was experienced).
- `Drawdown high` / `Drawdown low`: Profit at the beginning and end of the largest drawdown period. A negative low value means initial capital lost.
@ -421,6 +430,7 @@ It contains some useful key metrics about performance of your strategy on backte
Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions:
- Buys happen at open-price
- All orders are filled at the requested price (no slippage, no unfilled orders)
- Sell-signal sells happen at open-price of the consecutive candle
- Sell-signal is favored over Stoploss, because sell-signals are assumed to trigger on candle's open
- ROI
@ -468,11 +478,11 @@ There will be an additional table comparing win/losses of the different strategi
Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy.
```
=========================================================== STRATEGY SUMMARY ===========================================================
| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses |
|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|
| Strategy1 | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 |
| Strategy2 | 1487 | -0.13 | -197.58 | -0.00988917 | -98.79 | 4:43:00 | 662 | 0 | 825 |
=========================================================== STRATEGY SUMMARY =========================================================================
| Strategy | Buys | Avg Profit % | Cum Profit % | Tot Profit BTC | Tot Profit % | Avg Duration | Wins | Draws | Losses | Drawdown % |
|:------------|-------:|---------------:|---------------:|-----------------:|---------------:|:---------------|------:|-------:|-------:|-----------:|
| Strategy1 | 429 | 0.36 | 152.41 | 0.00762792 | 76.20 | 4:12:00 | 186 | 0 | 243 | 45.2 |
| Strategy2 | 1487 | -0.13 | -197.58 | -0.00988917 | -98.79 | 4:43:00 | 662 | 0 | 825 | 241.68 |
```
## Next step

33
docs/configuration.md
View File

@ -11,7 +11,16 @@ Per default, the bot loads the configuration from the `config.json` file, locate
You can specify a different configuration file used by the bot with the `-c/--config` command line option.
In some advanced use cases, multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
!!! Tip "Use multiple configuration files to keep secrets secret"
You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself.
``` bash
freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>
```
The 2nd file should only specify what you intend to override.
If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`).
If you used the [Quick start](installation.md/#quick-start) method for installing
the bot, the installation script should have already created the default configuration file (`config.json`) for you.
@ -59,8 +68,9 @@ Mandatory parameters are marked as **Required**, which means that they are requi
| `trailing_stop_positive_offset` | Offset on when to apply `trailing_stop_positive`. Percentage value which should be positive. More details in the [stoploss documentation](stoploss.md#trailing-stop-loss-only-once-the-trade-has-reached-a-certain-offset). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `0.0` (no offset).* <br> **Datatype:** Float
| `trailing_only_offset_is_reached` | Only apply trailing stoploss when the offset is reached. [stoploss documentation](stoploss.md). [Strategy Override](#parameters-in-the-strategy). <br>*Defaults to `false`.* <br> **Datatype:** Boolean
| `fee` | Fee used during backtesting / dry-runs. Should normally not be configured, which has freqtrade fall back to the exchange default fee. Set as ratio (e.g. 0.001 = 0.1%). Fee is applied twice for each trade, once when buying, once when selling. <br> **Datatype:** Float (as ratio)
| `unfilledtimeout.buy` | **Required.** How long (in minutes) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
| `unfilledtimeout.sell` | **Required.** How long (in minutes) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
| `unfilledtimeout.buy` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled buy order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
| `unfilledtimeout.sell` | **Required.** How long (in minutes or seconds) the bot will wait for an unfilled sell order to complete, after which the order will be cancelled and repeated at current (new) price, as long as there is a signal. [Strategy Override](#parameters-in-the-strategy).<br> **Datatype:** Integer
| `unfilledtimeout.unit` | Unit to use in unfilledtimeout setting. Note: If you set unfilledtimeout.unit to "seconds", "internals.process_throttle_secs" must be inferior or equal to timeout [Strategy Override](#parameters-in-the-strategy). <br> *Defaults to `minutes`.* <br> **Datatype:** String
| `bid_strategy.price_side` | Select the side of the spread the bot should look at to get the buy rate. [More information below](#buy-price-side).<br> *Defaults to `bid`.* <br> **Datatype:** String (either `ask` or `bid`).
| `bid_strategy.ask_last_balance` | **Required.** Interpolate the bidding price. More information [below](#buy-price-without-orderbook-enabled).
| `bid_strategy.use_order_book` | Enable buying using the rates in [Order Book Bids](#buy-price-with-orderbook-enabled). <br> **Datatype:** Boolean
@ -167,7 +177,7 @@ This exchange has also a limit on USD - where all orders must be > 10$ - which h
To guarantee safe execution, freqtrade will not allow buying with a stake-amount of 10.1$, instead, it'll make sure that there's enough space to place a stoploss below the pair (+ an offset, defined by `amount_reserve_percent`, which defaults to 5%).
With a stoploss of 10% - we'd therefore end up with a value of ~13.8$ (`12 * (1 + 0.05 + 0.1)`).
With a reserve of 5%, the minimum stake amount would be ~12.6$ (`12 * (1 + 0.05)`). If we take in account a stoploss of 10% on top of that - we'd end up with a value of ~14$ (`12.6 / (1 - 0.1)`).
To limit this calculation in case of large stoploss values, the calculated minimum stake-limit will never be more than 50% above the real limit.
@ -518,16 +528,27 @@ API Keys are usually only required for live trading (trading for real money, bot
**Insert your Exchange API key (change them by fake api keys):**
```json
"exchange": {
{
"exchange": {
"name": "bittrex",
"key": "af8ddd35195e9dc500b9a6f799f6f5c93d89193b",
"secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5",
...
//"password": "", // Optional, not needed by all exchanges)
// ...
}
//...
}
```
You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange.
!!! Hint "Keep your secrets secret"
To keep your secrets secret, we recommend to use a 2nd configuration for your API keys.
Simply use the above snippet in a new configuration file (e.g. `config-private.json`) and keep your settings in this file.
You can then start the bot with `freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>` to have your keys loaded.
**NEVER** share your private configuration file or your exchange keys with anyone!
### Using proxy with Freqtrade
To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration.

18
docs/data-download.md
View File

@ -11,8 +11,9 @@ Otherwise `--exchange` becomes mandatory.
You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used.
!!! Tip "Tip: Updating existing data"
If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data.
Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded.
If you already have backtesting data available in your data-directory and would like to refresh this data up to today, do not use `--days` or `--timerange` parameters. Freqtrade will keep the available data and only download the missing data.
If you are updating existing data after inserting new pairs that you have no data for, use `--new-pairs-days xx` parameter. Specified number of days will be downloaded for new pairs while old pairs will be updated with missing data only.
If you use `--days xx` parameter alone - data for specified number of days will be downloaded for _all_ pairs. Be careful, if specified number of days is smaller than gap between now and last downloaded candle - freqtrade will delete all existing data to avoid gaps in candle data.
### Usage
@ -20,8 +21,9 @@ You can use a relative timerange (`--days 20`) or an absolute starting point (`-
usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[-d PATH] [--userdir PATH]
[-p PAIRS [PAIRS ...]] [--pairs-file FILE]
[--days INT] [--timerange TIMERANGE]
[--dl-trades] [--exchange EXCHANGE]
[--days INT] [--new-pairs-days INT]
[--timerange TIMERANGE] [--dl-trades]
[--exchange EXCHANGE]
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
[--erase]
[--data-format-ohlcv {json,jsongz,hdf5}]
@ -30,10 +32,12 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
optional arguments:
-h, --help show this help message and exit
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
Show profits for only these pairs. Pairs are space-
Limit command to these pairs. Pairs are space-
separated.
--pairs-file FILE File containing a list of pairs to download.
--days INT Download data for given number of days.
--new-pairs-days INT Download data of new pairs for given number of days.
Default: `None`.
--timerange TIMERANGE
Specify what timerange of data to use.
--dl-trades Download trades instead of OHLCV data. The bot will
@ -48,10 +52,10 @@ optional arguments:
exchange/pairs/timeframes.
--data-format-ohlcv {json,jsongz,hdf5}
Storage format for downloaded candle (OHLCV) data.
(default: `json`).
(default: `None`).
--data-format-trades {json,jsongz,hdf5}
Storage format for downloaded trades data. (default:
`jsongz`).
`None`).
Common arguments:
-v, --verbose Verbose mode (-vv for more, -vvv to get all messages).

46
docs/docker_quickstart.md
View File

@ -10,11 +10,11 @@ Start by downloading and installing Docker CE for your platform:
* [Windows](https://docs.docker.com/docker-for-windows/install/)
* [Linux](https://docs.docker.com/install/)
To simplify running freqtrade, please install [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start).
To simplify running freqtrade, [`docker-compose`](https://docs.docker.com/compose/install/) should be installed and available to follow the below [docker quick start guide](#docker-quick-start).
## Freqtrade with docker-compose
Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage.
Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/stable/docker-compose.yml) ready for usage.
!!! Note
- The following section assumes that `docker` and `docker-compose` are installed and available to the logged in user.
@ -22,7 +22,7 @@ Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.co
### Docker quick start
Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory.
Create a new directory and place the [docker-compose file](https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml) in this directory.
=== "PC/MAC/Linux"
``` bash
@ -48,6 +48,8 @@ Create a new directory and place the [docker-compose file](https://github.com/fr
# Download the docker-compose file from the repository
curl https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml -o docker-compose.yml
# Edit the compose file to use an image named `*_pi` (stable_pi or develop_pi)
# Pull the freqtrade image
docker-compose pull
@ -65,6 +67,40 @@ Create a new directory and place the [docker-compose file](https://github.com/fr
# image: freqtradeorg/freqtrade:develop_pi
```
=== "ARM 64 Systenms (Mac M1, Raspberry Pi 4, Jetson Nano)"
In case of a Mac M1, make sure that your docker installation is running in native mode
Arm64 images are not yet provided via Docker Hub and need to be build locally first.
Depending on the device, this may take a few minutes (Apple M1) or multiple hours (Raspberry Pi)
``` bash
# Clone Freqtrade repository
git clone https://github.com/freqtrade/freqtrade.git
cd freqtrade
# Optionally switch to the stable version
git checkout stable
# Modify your docker-compose file to enable building and change the image name
# (see the Note Box below for necessary changes)
# Build image
docker-compose build
# Create user directory structure
docker-compose run --rm freqtrade create-userdir --userdir user_data
# Create configuration - Requires answering interactive questions
docker-compose run --rm freqtrade new-config --config user_data/config.json
```
!!! Note "Change your docker Image"
You have to change the docker image in the docker-compose file for your arm64 build to work properly.
``` yml
image: freqtradeorg/freqtrade:custom_arm64
build:
context: .
dockerfile: "./docker/Dockerfile.aarch64"
```
The above snippet creates a new directory called `ft_userdata`, downloads the latest compose file and pulls the freqtrade image.
The last 2 steps in the snippet create the directory with `user_data`, as well as (interactively) the default configuration based on your selections.
@ -156,8 +192,8 @@ Head over to the [Backtesting Documentation](backtesting.md) to learn more.
### Additional dependencies with docker-compose
If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host.
For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) for an example).
If your strategy requires dependencies not included in the default image - it will be necessary to build the image on your host.
For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.custom](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.custom) for an example).
You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions.

19
docs/edge.md
View File

@ -1,9 +1,9 @@
# Edge positioning
The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss.
The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss.
!!! Warning
`Edge positioning` is not compatible with dynamic (volume-based) whitelist.
WHen using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data.
!!! Note
`Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file.
@ -14,7 +14,7 @@ The `Edge Positioning` module uses probability to calculate your win rate and ri
Trading strategies are not perfect. They are frameworks that are susceptible to the market and its indicators. Because the market is not at all predictable, sometimes a strategy will win and sometimes the same strategy will lose.
To obtain an edge in the market, a strategy has to make more money than it loses. Making money in trading is not only about *how often* the strategy makes or loses money.
To obtain an edge in the market, a strategy has to make more money than it loses. Making money in trading is not only about *how often* the strategy makes or loses money.
!!! tip "It doesn't matter how often, but how much!"
A bad strategy might make 1 penny in *ten* transactions but lose 1 dollar in *one* transaction. If one only checks the number of winning trades, it would be misleading to think that the strategy is actually making a profit.
@ -215,16 +215,20 @@ Let's say the stake currency is **ETH** and there is $10$ **ETH** on the wallet.
usage: freqtrade edge [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
[--userdir PATH] [-s NAME] [--strategy-path PATH]
[-i TIMEFRAME] [--timerange TIMERANGE]
[--data-format-ohlcv {json,jsongz,hdf5}]
[--max-open-trades INT] [--stake-amount STAKE_AMOUNT]
[--fee FLOAT] [--stoplosses STOPLOSS_RANGE]
[--fee FLOAT] [-p PAIRS [PAIRS ...]]
[--stoplosses STOPLOSS_RANGE]
optional arguments:
-h, --help show this help message and exit
-i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
`1d`).
Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
--timerange TIMERANGE
Specify what timerange of data to use.
--data-format-ohlcv {json,jsongz,hdf5}
Storage format for downloaded candle (OHLCV) data.
(default: `None`).
--max-open-trades INT
Override the value of the `max_open_trades`
configuration setting.
@ -233,6 +237,9 @@ optional arguments:
setting.
--fee FLOAT Specify fee ratio. Will be applied twice (on trade
entry and exit).
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
Limit command to these pairs. Pairs are space-
separated.
--stoplosses STOPLOSS_RANGE
Defines a range of stoploss values against which edge
will assess the strategy. The format is "min,max,step"

25
docs/exchanges.md
View File

@ -7,10 +7,10 @@ This page combines common gotchas and informations which are exchange-specific a
!!! Tip "Stoploss on Exchange"
Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it.
### Blacklists
### Binance Blacklist
For Binance, please add `"BNB/<STAKE>"` to your blacklist to avoid issues.
Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB order unsellable as the expected amount is not there anymore.
Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB trade unsellable as the expected amount is not there anymore.
### Binance sites
@ -44,6 +44,10 @@ Due to the heavy rate-limiting applied by Kraken, the following configuration se
Downloading kraken data will require significantly more memory (RAM) than any other exchange, as the trades-data needs to be converted into candles on your machine.
It will also take a long time, as freqtrade will need to download every single trade that happened on the exchange for the pair / timerange combination, therefore please be patient.
!!! Warning "rateLimit tuning"
Please pay attention that rateLimit configuration entry holds delay in milliseconds between requests, NOT requests\sec rate.
So, in order to mitigate Kraken API "Rate limit exceeded" exception, this configuration should be increased, NOT decreased.
## Bittrex
### Order types
@ -96,6 +100,23 @@ To use subaccounts with FTX, you need to edit the configuration and add the foll
}
```
## Kucoin
Kucoin requries a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
```json
"exchange": {
"name": "kucoin",
"key": "your_exchange_key",
"secret": "your_exchange_secret",
"password": "your_exchange_api_key_password",
```
### Kucoin Blacklists
For Kucoin, please add `"KCS/<STAKE>"` to your blacklist to avoid issues.
Accounts having KCS accounts use this to pay for fees - if your first trade happens to be on `KCS`, further trades will consume this position and make the initial KCS trade unsellable as the expected amount is not there anymore.
## All exchanges
Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys.

16
docs/faq.md
View File

@ -1,5 +1,19 @@
# Freqtrade FAQ
## Supported Markets
Freqtrade supports spot trading only.
### Can I open short positions?
No, Freqtrade does not support trading with margin / leverage, and cannot open short positions.
In some cases, your exchange may provide leveraged spot tokens which can be traded with Freqtrade eg. BTCUP/USD, BTCDOWN/USD, ETHBULL/USD, ETHBEAR/USD, etc...
### Can I trade options or futures?
No, options and futures trading are not supported.
## Beginner Tips & Tricks
* When you work with your strategy & hyperopt file you should use a proper code editor like VSCode or PyCharm. A good code editor will provide syntax highlighting as well as line numbers, making it easy to find syntax errors (most likely pointed out by Freqtrade during startup).
@ -142,7 +156,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD
### Why does it take a long time to run hyperopt?
* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/MA9v74M). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
* If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers:

554
docs/hyperopt.md
View File

@ -1,19 +1,22 @@
# Hyperopt
This page explains how to tune your strategy by finding the optimal
parameters, a process called hyperparameter optimization. The bot uses several
algorithms included in the `scikit-optimize` package to accomplish this. The
search will burn all your CPU cores, make your laptop sound like a fighter jet
and still take a long time.
parameters, a process called hyperparameter optimization. The bot uses algorithms included in the `scikit-optimize` package to accomplish this.
The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time.
In general, the search for best parameters starts with a few random combinations (see [below](#reproducible-results) for more details) and then uses Bayesian search with a ML regressor algorithm (currently ExtraTreesRegressor) to quickly find a combination of parameters in the search hyperspace that minimizes the value of the [loss function](#loss-functions).
Hyperopt requires historic data to be available, just as backtesting does.
Hyperopt requires historic data to be available, just as backtesting does (hyperopt runs backtesting many times with different parameters).
To learn how to get data for the pairs and exchange you're interested in, head over to the [Data Downloading](data-download.md) section of the documentation.
!!! Bug
Hyperopt can crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133)
!!! Note
Since 2021.4 release you no longer have to write a separate hyperopt class, but can configure the parameters directly in the strategy.
The legacy method is still supported, but it is no longer the recommended way of setting up hyperopt.
The legacy documentation is available at [Legacy Hyperopt](advanced-hyperopt.md#legacy-hyperopt).
## Install hyperopt dependencies
Since Hyperopt dependencies are not needed to run the bot itself, are heavy, can not be easily built on some platforms (like Raspberry PI), they are not installed by default. Before you run Hyperopt, you need to install the corresponding dependencies, as described in this section below.
@ -34,7 +37,6 @@ pip install -r requirements-hyperopt.txt
## Hyperopt command reference
```
usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
[--userdir PATH] [-s NAME] [--strategy-path PATH]
@ -42,8 +44,9 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
[--data-format-ohlcv {json,jsongz,hdf5}]
[--max-open-trades INT]
[--stake-amount STAKE_AMOUNT] [--fee FLOAT]
[--hyperopt NAME] [--hyperopt-path PATH] [--eps]
[--dmmp] [--enable-protections]
[-p PAIRS [PAIRS ...]] [--hyperopt NAME]
[--hyperopt-path PATH] [--eps] [--dmmp]
[--enable-protections]
[--dry-run-wallet DRY_RUN_WALLET] [-e INT]
[--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]]
[--print-all] [--no-color] [--print-json] [-j JOBS]
@ -53,8 +56,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
optional arguments:
-h, --help show this help message and exit
-i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
`1d`).
Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
--timerange TIMERANGE
Specify what timerange of data to use.
--data-format-ohlcv {json,jsongz,hdf5}
@ -68,6 +70,9 @@ optional arguments:
setting.
--fee FLOAT Specify fee ratio. Will be applied twice (on trade
entry and exit).
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
Limit command to these pairs. Pairs are space-
separated.
--hyperopt NAME Specify hyperopt class name which will be used by the
bot.
--hyperopt-path PATH Specify additional lookup path for Hyperopt and
@ -104,7 +109,8 @@ optional arguments:
reproducible hyperopt results.
--min-trades INT Set minimal desired number of trades for evaluations
in the hyperopt optimization path (default: 1).
--hyperopt-loss NAME Specify the class name of the hyperopt loss function
--hyperopt-loss NAME, --hyperoptloss NAME
Specify the class name of the hyperopt loss function
class (IHyperOptLoss). Different functions can
generate completely different results, since the
target for optimization is different. Built-in
@ -137,47 +143,19 @@ Strategy arguments:
```
## Prepare Hyperopting
Before we start digging into Hyperopt, we recommend you to take a look at
the sample hyperopt file located in [user_data/hyperopts/](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt.py).
Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar.
!!! Tip "About this page"
For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class.
The simplest way to get started is to use the following, command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`.
``` bash
freqtrade new-hyperopt --hyperopt AwesomeHyperopt
```
### Hyperopt checklist
Checklist on all tasks / possibilities in hyperopt
Depending on the space you want to optimize, only some of the below are required:
* fill `buy_strategy_generator` - for buy signal optimization
* fill `indicator_space` - for buy signal optimization
* fill `sell_strategy_generator` - for sell signal optimization
* fill `sell_indicator_space` - for sell signal optimization
* define parameters with `space='buy'` - for buy signal optimization
* define parameters with `space='sell'` - for sell signal optimization
!!! Note
`populate_indicators` needs to create all indicators any of the spaces may use, otherwise hyperopt will not work.
Optional in hyperopt - can also be loaded from a strategy (recommended):
* `populate_indicators` - fallback to create indicators
* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy
* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy
!!! Note
You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods.
Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead.
Rarely you may also need to override:
Rarely you may also need to create a [nested class](advanced-hyperopt.md#overriding-pre-defined-spaces) named `HyperOpt` and implement
* `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default)
* `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps)
@ -185,31 +163,30 @@ Rarely you may also need to override:
* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default)
!!! Tip "Quickly optimize ROI, stoploss and trailing stoploss"
You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything (i.e. without creation of a "complete" Hyperopt class with dimensions, parameters, triggers and guards, as described in this document) from the default hyperopt template by relying on your strategy to do most of the calculations.
You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything in your strategy.
```python
``` bash
# Have a working strategy at hand.
freqtrade new-hyperopt --hyperopt EmptyHyperopt
freqtrade hyperopt --hyperopt EmptyHyperopt --hyperopt-loss SharpeHyperOptLossDaily --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100
freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100
```
### Create a Custom Hyperopt File
### Hyperopt execution logic
Let assume you want a hyperopt file `AwesomeHyperopt.py`:
Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators.
``` bash
freqtrade new-hyperopt --hyperopt AwesomeHyperopt
```
Hyperopt will then spawn into different processes (number of processors, or `-j <n>`), and run backtesting over and over again, changing the parameters that are part of the `--spaces` defined.
This command will create a new hyperopt file from a template, allowing you to get started quickly.
For every new set of parameters, freqtrade will run first `populate_buy_trend()` followed by `populate_sell_trend()`, and then run the regular backtesting process to simulate trades.
After backtesting, the results are passed into the [loss function](#loss-functions), which will evaluate if this result was better or worse than previous results.
Based on the loss function result, hyperopt will determine the next set of parameters to try in the next round of backtesting.
### Configure your Guards and Triggers
There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing:
There are two places you need to change in your strategy file to add a new buy hyperopt for testing:
* Inside `indicator_space()` - the parameters hyperopt shall be optimizing.
* Within `buy_strategy_generator()` - populate the nested `populate_buy_trend()` to apply the parameters.
* Define the parameters at the class level hyperopt shall be optimizing.
* Within `populate_buy_trend()` - use defined parameter values instead of raw constants.
There you have two different types of indicators: 1. `guards` and 2. `triggers`.
@ -221,100 +198,106 @@ There you have two different types of indicators: 1. `guards` and 2. `triggers`.
However, this guide will make this distinction to make it clear that signals should not be "sticking".
Sticking signals are signals that are active for multiple candles. This can lead into buying a signal late (right before the signal disappears - which means that the chance of success is a lot lower than right at the beginning).
Hyper-optimization will, for each epoch round, pick one trigger and possibly
multiple guards. The constructed strategy will be something like "*buy exactly when close price touches lower Bollinger band, BUT only if
ADX > 10*".
If you have updated the buy strategy, i.e. changed the contents of `populate_buy_trend()` method, you have to update the `guards` and `triggers` your hyperopt must use correspondingly.
Hyper-optimization will, for each epoch round, pick one trigger and possibly multiple guards.
#### Sell optimization
Similar to the buy-signal above, sell-signals can also be optimized.
Place the corresponding settings into the following methods
* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing.
* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters.
* Define the parameters at the class level hyperopt shall be optimizing, either naming them `sell_*`, or by explicitly defining `space='sell'`.
* Within `populate_sell_trend()` - use defined parameter values instead of raw constants.
The configuration and rules are the same than for buy signals.
To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`.
#### Using timeframe as a part of the Strategy
The Strategy class exposes the timeframe value as the `self.timeframe` attribute.
The same value is available as class-attribute `HyperoptName.timeframe`.
In the case of the linked sample-value this would be `AwesomeHyperopt.timeframe`.
## Solving a Mystery
Let's say you are curious: should you use MACD crossings or lower Bollinger
Bands to trigger your buys. And you also wonder should you use RSI or ADX to
help with those buy decisions. If you decide to use RSI or ADX, which values
should I use for them? So let's use hyperparameter optimization to solve this
mystery.
Let's say you are curious: should you use MACD crossings or lower Bollinger Bands to trigger your buys.
And you also wonder should you use RSI or ADX to help with those buy decisions.
If you decide to use RSI or ADX, which values should I use for them?
We will start by defining a search space:
So let's use hyperparameter optimization to solve this mystery.
```python
def indicator_space() -> List[Dimension]:
### Defining indicators to be used
We start by calculating the indicators our strategy is going to use.
``` python
class MyAwesomeStrategy(IStrategy):
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""
Define your Hyperopt space for searching strategy parameters
Generate all indicators used by the strategy
"""
return [
Integer(20, 40, name='adx-value'),
Integer(20, 40, name='rsi-value'),
Categorical([True, False], name='adx-enabled'),
Categorical([True, False], name='rsi-enabled'),
Categorical(['bb_lower', 'macd_cross_signal'], name='trigger')
]
dataframe['adx'] = ta.ADX(dataframe)
dataframe['rsi'] = ta.RSI(dataframe)
macd = ta.MACD(dataframe)
dataframe['macd'] = macd['macd']
dataframe['macdsignal'] = macd['macdsignal']
dataframe['macdhist'] = macd['macdhist']
bollinger = ta.BBANDS(dataframe, timeperiod=20, nbdevup=2.0, nbdevdn=2.0)
dataframe['bb_lowerband'] = boll['lowerband']
dataframe['bb_middleband'] = boll['middleband']
dataframe['bb_upperband'] = boll['upperband']
return dataframe
```
Above definition says: I have five parameters I want you to randomly combine
to find the best combination. Two of them are integer values (`adx-value`
and `rsi-value`) and I want you test in the range of values 20 to 40.
### Hyperoptable parameters
We continue to define hyperoptable parameters:
```python
class MyAwesomeStrategy(IStrategy):
buy_adx = IntParameter(20, 40, default=30, space="buy")
buy_rsi = IntParameter(20, 40, default=30, space="buy")
buy_adx_enabled = CategoricalParameter([True, False], space="buy")
buy_rsi_enabled = CategoricalParameter([True, False], space="buy")
buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal'], space="buy")
```
Above definition says: I have five parameters I want to randomly combine to find the best combination.
Two of them are integer values (`buy_adx` and `buy_rsi`) and I want you test in the range of values 20 to 40.
Then we have three category variables. First two are either `True` or `False`.
We use these to either enable or disable the ADX and RSI guards. The last
one we call `trigger` and use it to decide which buy trigger we want to use.
We use these to either enable or disable the ADX and RSI guards.
The last one we call `trigger` and use it to decide which buy trigger we want to use.
!!! Note "Parameter space assignment"
Parameters must either be assigned to a variable named `buy_*` or `sell_*` - or contain `space='buy'` | `space='sell'` to be assigned to a space correctly.
If no parameter is available for a space, you'll receive the error that no space was found when running hyperopt.
So let's write the buy strategy using these values:
```python
@staticmethod
def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
"""
Define the buy strategy parameters to be used by Hyperopt.
"""
def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
conditions = []
# GUARDS AND TRENDS
if 'adx-enabled' in params and params['adx-enabled']:
conditions.append(dataframe['adx'] > params['adx-value'])
if 'rsi-enabled' in params and params['rsi-enabled']:
conditions.append(dataframe['rsi'] < params['rsi-value'])
def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
conditions = []
# GUARDS AND TRENDS
if self.buy_adx_enabled.value:
conditions.append(dataframe['adx'] > self.buy_adx.value)
if self.buy_rsi_enabled.value:
conditions.append(dataframe['rsi'] < self.buy_rsi.value)
# TRIGGERS
if 'trigger' in params:
if params['trigger'] == 'bb_lower':
conditions.append(dataframe['close'] < dataframe['bb_lowerband'])
if params['trigger'] == 'macd_cross_signal':
conditions.append(qtpylib.crossed_above(
dataframe['macd'], dataframe['macdsignal']
))
# TRIGGERS
if self.buy_trigger.value == 'bb_lower':
conditions.append(dataframe['close'] < dataframe['bb_lowerband'])
if self.buy_trigger.value == 'macd_cross_signal':
conditions.append(qtpylib.crossed_above(
dataframe['macd'], dataframe['macdsignal']
))
# Check that volume is not 0
conditions.append(dataframe['volume'] > 0)
# Check that volume is not 0
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
'buy'] = 1
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
'buy'] = 1
return dataframe
return populate_buy_trend
return dataframe
```
Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations.
It will use the given historical data and make buys based on the buy signals generated with the above function.
It will use the given historical data and simulate buys based on the buy signals generated with the above function.
Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)).
!!! Note
@ -322,6 +305,108 @@ Based on the results, hyperopt will tell you which parameter combination produce
When you want to test an indicator that isn't used by the bot currently, remember to
add it to the `populate_indicators()` method in your strategy or hyperopt file.
## Parameter types
There are four parameter types each suited for different purposes.
* `IntParameter` - defines an integral parameter with upper and lower boundaries of search space.
* `DecimalParameter` - defines a floating point parameter with a limited number of decimals (default 3). Should be preferred instead of `RealParameter` in most cases.
* `RealParameter` - defines a floating point parameter with upper and lower boundaries and no precision limit. Rarely used as it creates a space with a near infinite number of possibilities.
* `CategoricalParameter` - defines a parameter with a predetermined number of choices.
!!! Tip "Disabling parameter optimization"
Each parameter takes two boolean parameters:
* `load` - when set to `False` it will not load values configured in `buy_params` and `sell_params`.
* `optimize` - when set to `False` parameter will not be included in optimization process.
Use these parameters to quickly prototype various ideas.
!!! Warning
Hyperoptable parameters cannot be used in `populate_indicators` - as hyperopt does not recalculate indicators for each epoch, so the starting value would be used in this case.
### Optimizing an indicator parameter
Assuming you have a simple strategy in mind - a EMA cross strategy (2 Moving averages crossing) - and you'd like to find the ideal parameters for this strategy.
``` python
from pandas import DataFrame
from functools import reduce
import talib.abstract as ta
from freqtrade.strategy import IStrategy
from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter
import freqtrade.vendor.qtpylib.indicators as qtpylib
class MyAwesomeStrategy(IStrategy):
stoploss = -0.05
timeframe = '15m'
# Define the parameter spaces
buy_ema_short = IntParameter(3, 50, default=5)
buy_ema_long = IntParameter(15, 200, default=50)
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
"""Generate all indicators used by the strategy"""
# Calculate all ema_short values
for val in self.buy_ema_short.range:
dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val)
# Calculate all ema_long values
for val in self.buy_ema_long.range:
dataframe[f'ema_long_{val}'] = ta.EMA(dataframe, timeperiod=val)
return dataframe
def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
conditions = []
conditions.append(qtpylib.crossed_above(
dataframe[f'ema_short_{self.buy_ema_short.value}'], dataframe[f'ema_long_{self.buy_ema_long.value}']
))
# Check that volume is not 0
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
'buy'] = 1
return dataframe
def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
conditions = []
conditions.append(qtpylib.crossed_above(
dataframe[f'ema_long_{self.buy_ema_long.value}'], dataframe[f'ema_short_{self.buy_ema_short.value}']
))
# Check that volume is not 0
conditions.append(dataframe['volume'] > 0)
if conditions:
dataframe.loc[
reduce(lambda x, y: x & y, conditions),
'sell'] = 1
return dataframe
```
Breaking it down:
Using `self.buy_ema_short.range` will return a range object containing all entries between the Parameters low and high value.
In this case (`IntParameter(3, 50, default=5)`), the loop would run for all numbers between 3 and 50 (`[3, 4, 5, ... 49, 50]`).
By using this in a loop, hyperopt will generate 48 new columns (`['buy_ema_3', 'buy_ema_4', ... , 'buy_ema_50']`).
Hyperopt itself will then use the selected value to create the buy and sell signals
While this strategy is most likely too simple to provide consistent profit, it should serve as an example how optimize indicator parameters.
!!! Note
`self.buy_ema_short.range` will act differently between hyperopt and other modes. For hyperopt, the above example may generate 48 new columns, however for all other modes (backtesting, dry/live), it will only generate the column for the selected value. You should therefore avoid using the resulting column with explicit values (values other than `self.buy_ema_short.value`).
??? Hint "Performance tip"
By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter.
While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values).
You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space.
## Loss-functions
Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results.
@ -343,16 +428,14 @@ Creation of a custom loss function is covered in the [Advanced Hyperopt](advance
## Execute Hyperopt
Once you have updated your hyperopt configuration you can run it.
Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. More time usually results in better results.
Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result.
We strongly recommend to use `screen` or `tmux` to prevent any connection loss.
```bash
freqtrade hyperopt --config config.json --hyperopt <hyperoptname> --hyperopt-loss <hyperoptlossname> --strategy <strategyname> -e 500 --spaces all
freqtrade hyperopt --config config.json --hyperopt-loss <hyperoptlossname> --strategy <strategyname> -e 500 --spaces all
```
Use `<hyperoptname>` as the name of the custom hyperopt used.
The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs.
Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results.
@ -366,30 +449,23 @@ The `--spaces all` option determines that all possible parameters should be opti
### Execute Hyperopt with different historical data source
If you would like to hyperopt parameters using an alternate historical data set that
you have on-disk, use the `--datadir PATH` option. By default, hyperopt
uses data from directory `user_data/data`.
you have on-disk, use the `--datadir PATH` option. By default, hyperopt uses data from directory `user_data/data`.
### Running Hyperopt with a smaller test-set
Use the `--timerange` argument to change how much of the test-set you want to use.
For example, to use one month of data, pass the following parameter to the hyperopt call:
For example, to use one month of data, pass `--timerange 20210101-20210201` (from january 2021 - february 2021) to the hyperopt call.
Full command:
```bash
freqtrade hyperopt --hyperopt <hyperoptname> --strategy <strategyname> --timerange 20180401-20180501
```
### Running Hyperopt using methods from a strategy
Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided.
```bash
freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy
freqtrade hyperopt --hyperopt <hyperoptname> --strategy <strategyname> --timerange 20210101-20210201
```
### Running Hyperopt with Smaller Search Space
Use the `--spaces` option to limit the search space used by hyperopt.
Letting Hyperopt optimize everything is a huuuuge search space.
Letting Hyperopt optimize everything is a huuuuge search space.
Often it might make more sense to start by just searching for initial buy algorithm.
Or maybe you just want to optimize your stoploss or roi table for that awesome new buy strategy you have.
@ -406,40 +482,9 @@ Legal values are:
The default Hyperopt Search Space, used when no `--space` command line option is specified, does not include the `trailing` hyperspace. We recommend you to run optimization for the `trailing` hyperspace separately, when the best parameters for other hyperspaces were found, validated and pasted into your custom strategy.
### Position stacking and disabling max market positions
In some situations, you may need to run Hyperopt (and Backtesting) with the
`--eps`/`--enable-position-staking` and `--dmmp`/`--disable-max-market-positions` arguments.
By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one
open trade is allowed for every traded pair. The total number of trades open for all pairs
is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to
some potential trades to be hidden (or masked) by previously open trades.
The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times,
while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades`
during Hyperopt/Backtesting (which is equal to setting `max_open_trades` to a very high
number).
!!! Note
Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality.
You can also enable position stacking in the configuration file by explicitly setting
`"position_stacking"=true`.
### Reproducible results
The search for optimal parameters starts with a few (currently 30) random combinations in the hyperspace of parameters, random Hyperopt epochs. These random epochs are marked with an asterisk character (`*`) in the first column in the Hyperopt output.
The initial state for generation of these random values (random state) is controlled by the value of the `--random-state` command line option. You can set it to some arbitrary value of your choice to obtain reproducible results.
If you have not set this value explicitly in the command line options, Hyperopt seeds the random state with some random value for you. The random state value for each Hyperopt run is shown in the log, so you can copy and paste it into the `--random-state` command line option to repeat the set of the initial random epochs used.
If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyper-optimization results with same random state value used.
## Understand the Hyperopt Result
Once Hyperopt is completed you can use the result to create a new strategy.
Once Hyperopt is completed you can use the result to update your strategy.
Given the following result from hyperopt:
```
@ -447,49 +492,38 @@ Best result:
44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
Buy hyperspace params:
{ 'adx-value': 44,
'rsi-value': 29,
'adx-enabled': False,
'rsi-enabled': True,
'trigger': 'bb_lower'}
# Buy hyperspace params:
buy_params = {
'buy_adx': 44,
'buy_rsi': 29,
'buy_adx_enabled': False,
'buy_rsi_enabled': True,
'buy_trigger': 'bb_lower'
}
```
You should understand this result like:
- The buy trigger that worked best was `bb_lower`.
- You should not use ADX because `adx-enabled: False`)
- You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`)
* The buy trigger that worked best was `bb_lower`.
* You should not use ADX because `'buy_adx_enabled': False`.
* You should **consider** using the RSI indicator (`'buy_rsi_enabled': True`) and the best value is `29.0` (`'buy_rsi': 29.0`)
You have to look inside your strategy file into `buy_strategy_generator()`
method, what those values match to.
Your strategy class can immediately take advantage of these results. Simply copy hyperopt results block and paste them at class level, replacing old parameters (if any). New parameters will automatically be loaded next time strategy is executed.
So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block:
Transferring your whole hyperopt result to your strategy would then look like:
```python
(dataframe['rsi'] < 29.0)
class MyAwesomeStrategy(IStrategy):
# Buy hyperspace params:
buy_params = {
'buy_adx': 44,
'buy_rsi': 29,
'buy_adx_enabled': False,
'buy_rsi_enabled': True,
'buy_trigger': 'bb_lower'
}
```
Translating your whole hyperopt result as the new buy-signal would then look like:
```python
def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
dataframe.loc[
(
(dataframe['rsi'] < 29.0) & # rsi-value
dataframe['close'] < dataframe['bb_lowerband'] # trigger
),
'buy'] = 1
return dataframe
```
By default, hyperopt prints colorized results -- epochs with positive profit are printed in the green color. This highlighting helps you find epochs that can be interesting for later analysis. Epochs with zero total profit or with negative profits (losses) are printed in the normal color. If you do not need colorization of results (for instance, when you are redirecting hyperopt output to a file) you can switch colorization off by specifying the `--no-color` option in the command line.
You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option.
!!! Note "Windows and color output"
Windows does not support color-output natively, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL.
### Understand Hyperopt ROI results
If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table:
@ -499,11 +533,13 @@ Best result:
44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
ROI table:
{ 0: 0.10674,
21: 0.09158,
78: 0.03634,
118: 0}
# ROI table:
minimal_roi = {
0: 0.10674,
21: 0.09158,
78: 0.03634,
118: 0
}
```
In order to use this best ROI table found by Hyperopt in backtesting and for live trades/dry-run, copy-paste it as the value of the `minimal_roi` attribute of your custom strategy:
@ -523,23 +559,26 @@ As stated in the comment, you can also use it as the value of the `minimal_roi`
#### Default ROI Search Space
If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the timeframe used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 5 digits after the decimal point):
If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the timeframe used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 3 digits after the decimal point):
| # step | 1m | | 5m | | 1h | | 1d | |
| ------ | ------ | ----------------- | -------- | ----------- | ---------- | ----------------- | ------------ | ----------------- |
| 1 | 0 | 0.01161...0.11992 | 0 | 0.03...0.31 | 0 | 0.06883...0.71124 | 0 | 0.12178...1.25835 |
| 2 | 2...8 | 0.00774...0.04255 | 10...40 | 0.02...0.11 | 120...480 | 0.04589...0.25238 | 2880...11520 | 0.08118...0.44651 |
| 3 | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 |
| 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 |
| # step | 1m | | 5m | | 1h | | 1d | |
| ------ | ------ | ------------- | -------- | ----------- | ---------- | ------------- | ------------ | ------------- |
| 1 | 0 | 0.011...0.119 | 0 | 0.03...0.31 | 0 | 0.068...0.711 | 0 | 0.121...1.258 |
| 2 | 2...8 | 0.007...0.042 | 10...40 | 0.02...0.11 | 120...480 | 0.045...0.252 | 2880...11520 | 0.081...0.446 |
| 3 | 4...20 | 0.003...0.015 | 20...100 | 0.01...0.04 | 240...1200 | 0.022...0.091 | 5760...28800 | 0.040...0.162 |
| 4 | 6...44 | 0.0 | 30...220 | 0.0 | 360...2640 | 0.0 | 8640...63360 | 0.0 |
These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used.
If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default.
Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps).
Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps).
A sample for these methods can be found in [sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py).
!!! Note "Reduced search space"
To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs.
### Understand Hyperopt Stoploss results
If you are optimizing stoploss values (i.e. if optimization search-space contains 'all', 'default' or 'stoploss'), your result will look as follows and include stoploss:
@ -549,13 +588,16 @@ Best result:
44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
Buy hyperspace params:
{ 'adx-value': 44,
'rsi-value': 29,
'adx-enabled': False,
'rsi-enabled': True,
'trigger': 'bb_lower'}
Stoploss: -0.27996
# Buy hyperspace params:
buy_params = {
'buy_adx': 44,
'buy_rsi': 29,
'buy_adx_enabled': False,
'buy_rsi_enabled': True,
'buy_trigger': 'bb_lower'
}
stoploss: -0.27996
```
In order to use this best stoploss value found by Hyperopt in backtesting and for live trades/dry-run, copy-paste it as the value of the `stoploss` attribute of your custom strategy:
@ -576,6 +618,9 @@ If you have the `stoploss_space()` method in your custom hyperopt file, remove i
Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py).
!!! Note "Reduced search space"
To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs.
### Understand Hyperopt Trailing Stop results
If you are optimizing trailing stop values (i.e. if optimization search-space contains 'all' or 'trailing'), your result will look as follows and include trailing stop parameters:
@ -585,11 +630,11 @@ Best result:
45/100: 606 trades. Avg profit 1.04%. Total profit 0.31555614 BTC ( 630.48Σ%). Avg duration 150.3 mins. Objective: -1.10161
Trailing stop:
{ 'trailing_only_offset_is_reached': True,
'trailing_stop': True,
'trailing_stop_positive': 0.02001,
'trailing_stop_positive_offset': 0.06038}
# Trailing stop:
trailing_stop = True
trailing_stop_positive = 0.02001
trailing_stop_positive_offset = 0.06038
trailing_only_offset_is_reached = True
```
In order to use these best trailing stop parameters found by Hyperopt in backtesting and for live trades/dry-run, copy-paste them as the values of the corresponding attributes of your custom strategy:
@ -611,6 +656,59 @@ If you are optimizing trailing stop values, Freqtrade creates the 'trailing' opt
Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py).
!!! Note "Reduced search space"
To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs.
### Reproducible results
The search for optimal parameters starts with a few (currently 30) random combinations in the hyperspace of parameters, random Hyperopt epochs. These random epochs are marked with an asterisk character (`*`) in the first column in the Hyperopt output.
The initial state for generation of these random values (random state) is controlled by the value of the `--random-state` command line option. You can set it to some arbitrary value of your choice to obtain reproducible results.
If you have not set this value explicitly in the command line options, Hyperopt seeds the random state with some random value for you. The random state value for each Hyperopt run is shown in the log, so you can copy and paste it into the `--random-state` command line option to repeat the set of the initial random epochs used.
If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyper-optimization results with same random state value used.
## Output formatting
By default, hyperopt prints colorized results -- epochs with positive profit are printed in the green color. This highlighting helps you find epochs that can be interesting for later analysis. Epochs with zero total profit or with negative profits (losses) are printed in the normal color. If you do not need colorization of results (for instance, when you are redirecting hyperopt output to a file) you can switch colorization off by specifying the `--no-color` option in the command line.
You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option.
!!! Note "Windows and color output"
Windows does not support color-output natively, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL.
## Position stacking and disabling max market positions
In some situations, you may need to run Hyperopt (and Backtesting) with the
`--eps`/`--enable-position-staking` and `--dmmp`/`--disable-max-market-positions` arguments.
By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one
open trade is allowed for every traded pair. The total number of trades open for all pairs
is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to
some potential trades to be hidden (or masked) by previously open trades.
The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times,
while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades`
during Hyperopt/Backtesting (which is equal to setting `max_open_trades` to a very high
number).
!!! Note
Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality.
You can also enable position stacking in the configuration file by explicitly setting
`"position_stacking"=true`.
## Out of Memory errors
As hyperopt consumes a lot of memory (the complete data needs to be in memory once per parallel backtesting process), it's likely that you run into "out of memory" errors.
To combat these, you have multiple options:
* reduce the amount of pairs
* reduce the timerange used (`--timerange <timerange>`)
* reduce the number of parallel processes (`-j <n>`)
* Increase the memory of your machine
## Show details of Hyperopt results
After you run Hyperopt for the desired amount of epochs, you can later list all results for analysis, select only best or profitable once, and show the details for any of the epochs previously evaluated. This can be done with the `hyperopt-list` and `hyperopt-show` sub-commands. The usage of these sub-commands is described in the [Utils](utils.md#list-hyperopt-results) chapter.

46
docs/includes/pairlists.md
View File

@ -4,7 +4,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade.
In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler).
Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler.
@ -29,6 +29,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
* [`ShuffleFilter`](#shufflefilter)
* [`SpreadFilter`](#spreadfilter)
* [`RangeStabilityFilter`](#rangestabilityfilter)
* [`VolatilityFilter`](#volatilityfilter)
!!! Tip "Testing pairlists"
Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly.
@ -59,6 +60,8 @@ When used in the chain of Pairlist Handlers in a non-leading position (after Sta
When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top assets from all available markets (with matching stake-currency) on the exchange.
The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
`VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library:
@ -89,6 +92,7 @@ This filter allows freqtrade to ignore pairs until they have been listed for at
#### PerformanceFilter
Sorts pairs by past trade performance, as follows:
1. Positive performance.
2. No closed trades yet.
3. Negative performance.
@ -108,6 +112,7 @@ The `PriceFilter` allows filtering of pairs by price. Currently the following pr
* `min_price`
* `max_price`
* `max_value`
* `low_price_ratio`
The `min_price` setting removes pairs where the price is below the specified price. This is useful if you wish to avoid trading very low-priced pairs.
@ -116,6 +121,11 @@ This option is disabled by default, and will only apply if set to > 0.
The `max_price` setting removes pairs where the price is above the specified price. This is useful if you wish to trade only low-priced pairs.
This option is disabled by default, and will only apply if set to > 0.
The `max_value` setting removes pairs where the minimum value change is above a specified value.
This is useful when an exchange has unbalanced limits. For example, if step-size = 1 (so you can only buy 1, or 2, or 3, but not 1.1 Coins) - and the price is pretty high (like 20$) as the coin has risen sharply since the last limit adaption.
As a result of the above, you can only buy for 20$, or 40$ - but not for 25$.
On exchanges that deduct fees from the receiving currency (e.g. FTX) - this can result in high value coins / amounts that are unsellable as the amount is slightly below the limit.
The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio.
This option is disabled by default, and will only apply if set to > 0.
@ -164,9 +174,32 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit
!!! Tip
This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
#### VolatilityFilter
Volatility is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatility of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)).
This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs.
In the below example:
If the volatility over the last 10 days is not in the range of 0.05-0.50, remove the pair from the whitelist. The filter is applied every 24h.
```json
"pairlists": [
{
"method": "VolatilityFilter",
"lookback_days": 10,
"min_volatility": 0.05,
"max_volatility": 0.50,
"refresh_period": 86400
}
]
```
### Full example of Pairlist Handlers
The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value.
The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#pricefilter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value.
```json
"exchange": {
@ -177,7 +210,7 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
{
"method": "VolumePairList",
"number_assets": 20,
"sort_key": "quoteVolume",
"sort_key": "quoteVolume"
},
{"method": "AgeFilter", "min_days_listed": 10},
{"method": "PrecisionFilter"},
@ -189,6 +222,13 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
"min_rate_of_change": 0.01,
"refresh_period": 1440
},
{
"method": "VolatilityFilter",
"lookback_days": 10,
"min_volatility": 0.05,
"max_volatility": 0.50,
"refresh_period": 86400
},
{"method": "ShuffleFilter", "seed": 42}
],
```

5
docs/index.md
View File

@ -1,4 +1,5 @@
# Freqtrade
![freqtrade](assets/freqtrade_poweredby.svg)
[![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/)
[![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop)
[![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability)
@ -39,7 +40,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
- [X] [Bittrex](https://bittrex.com/)
- [X] [FTX](https://ftx.com)
- [X] [Kraken](https://kraken.com/)
- [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
- [ ] [potentially many others through <img alt="ccxt" width="30px" src="assets/ccxt-logo.svg" />](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
### Community tested

4
docs/installation.md
View File

@ -60,7 +60,7 @@ OS Specific steps are listed first, the [Common](#common) section below is neces
sudo apt-get update
# install packages
sudo apt install -y python3-pip python3-venv python3-pandas python3-pip git
sudo apt install -y python3-pip python3-venv python3-pandas git
```
=== "RaspberryPi/Raspbian"
@ -269,7 +269,7 @@ git clone https://github.com/freqtrade/freqtrade.git
cd freqtrade
```
#### Freqtrade instal: Conda Environment
#### Freqtrade install: Conda Environment
Prepare conda-freqtrade environment, using file `environment.yml`, which exist in main freqtrade directory

12
docs/plotting.md
View File

@ -37,7 +37,7 @@ usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH]
optional arguments:
-h, --help show this help message and exit
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
Show profits for only these pairs. Pairs are space-
Limit command to these pairs. Pairs are space-
separated.
--indicators1 INDICATORS1 [INDICATORS1 ...]
Set indicators from your strategy you want in the
@ -66,8 +66,7 @@ optional arguments:
--timerange TIMERANGE
Specify what timerange of data to use.
-i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
`1d`).
Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
--no-trades Skip using trades from backtesting file and DB.
Common arguments:
@ -91,6 +90,7 @@ Strategy arguments:
Specify strategy class name which will be used by the
bot.
--strategy-path PATH Specify additional strategy lookup path.
```
Example:
@ -245,7 +245,7 @@ usage: freqtrade plot-profit [-h] [-v] [--logfile FILE] [-V] [-c PATH]
optional arguments:
-h, --help show this help message and exit
-p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
Show profits for only these pairs. Pairs are space-
Limit command to these pairs. Pairs are space-
separated.
--timerange TIMERANGE
Specify what timerange of data to use.
@ -264,8 +264,7 @@ optional arguments:
Specify the source for trades (Can be DB or file
(backtest file)) Default: file
-i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
`1d`).
Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
Common arguments:
-v, --verbose Verbose mode (-vv for more, -vvv to get all messages).
@ -288,6 +287,7 @@ Strategy arguments:
Specify strategy class name which will be used by the
bot.
--strategy-path PATH Specify additional strategy lookup path.
```
The `-p/--pairs` argument, can be used to limit the pairs that are considered for this calculation.

4
docs/requirements-docs.txt
View File

@ -1,3 +1,3 @@
mkdocs-material==7.0.6
mkdocs-material==7.1.4
mdx_truly_sane_lists==1.2
pymdown-extensions==8.1.1
pymdown-extensions==8.2

33
docs/rest-api.md
View File

@ -71,7 +71,10 @@ If you run your bot using docker, you'll need to have the bot listen to incoming
"api_server": {
"enabled": true,
"listen_ip_address": "0.0.0.0",
"listen_port": 8080
"listen_port": 8080,
"username": "Freqtrader",
"password": "SuperSecret1!",
//...
},
```
@ -106,7 +109,10 @@ By default, the script assumes `127.0.0.1` (localhost) and port `8080` to be use
"api_server": {
"enabled": true,
"listen_ip_address": "0.0.0.0",
"listen_port": 8080
"listen_port": 8080,
"username": "Freqtrader",
"password": "SuperSecret1!",
//...
}
}
```
@ -124,7 +130,8 @@ python3 scripts/rest_client.py --config rest_config.json <command> [optional par
| `stop` | Stops the trader.
| `stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
| `reload_config` | Reloads the configuration file.
| `trades` | List last trades.
| `trades` | List last trades. Limited to 500 trades per call.
| `trade/<tradeid>` | Get specific trade.
| `delete_trade <trade_id>` | Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange.
| `show_config` | Shows part of the current configuration with relevant settings to operation.
| `logs` | Shows last log messages.
@ -181,7 +188,7 @@ count
Return the amount of open trades.
daily
Return the amount of open trades.
Return the profits for each day, and amount of trades.
delete_lock
Delete (disable) lock from the database.
@ -214,7 +221,7 @@ locks
logs
Show latest logs.
:param limit: Limits log messages to the last <limit> logs. No limit to get all the trades.
:param limit: Limits log messages to the last <limit> logs. No limit to get the entire log.
pair_candles
Return live dataframe for <pair><timeframe>.
@ -234,6 +241,9 @@ pair_history
performance
Return the performance of the different coins.
ping
simple ping
plot_config
Return plot configuration if the strategy defines one.
@ -270,17 +280,22 @@ strategy
:param strategy: Strategy class name
trades
Return trades history.
trade
Return specific trade
:param limit: Limits trades to the X last trades. No limit to get all the trades.
:param trade_id: Specify which trade to get.
trades
Return trades history, sorted by id
:param limit: Limits trades to the X last trades. Max 500 trades.
:param offset: Offset by this amount of trades.
version
Return the version of the bot.
whitelist
Show the current whitelist.
```
### OpenAPI interface

31
docs/sql_cheatsheet.md
View File

@ -19,7 +19,7 @@ The freqtrade docker image does contain sqlite3, so you can edit the database wi
``` bash
docker-compose exec freqtrade /bin/bash
sqlite3 <databasefile>.sqlite
sqlite3 <database-file>.sqlite
```
## Open the DB
@ -99,3 +99,32 @@ DELETE FROM trades WHERE id = 31;
!!! Warning
This will remove this trade from the database. Please make sure you got the correct id and **NEVER** run this query without the `where` clause.
## Use a different database system
!!! Warning
By using one of the below database systems, you acknowledge that you know how to manage such a system. Freqtrade will not provide any support with setup or maintenance (or backups) of the below database systems.
### PostgreSQL
Freqtrade supports PostgreSQL by using SQLAlchemy, which supports multiple different database systems.
Installation:
`pip install psycopg2`
Usage:
`... --db-url postgresql+psycopg2://<username>:<password>@localhost:5432/<database>`
Freqtrade will automatically create the tables necessary upon startup.
If you're running different instances of Freqtrade, you must either setup one database per Instance or use different users / schemas for your connections.
### MariaDB / MySQL
Freqtrade supports MariaDB by using SQLAlchemy, which supports multiple different database systems.
Installation:
`pip install pymysql`
Usage:
`... --db-url mysql+pymysql://<username>:<password>@localhost:3306/<database>`

135
docs/strategy-advanced.md
View File

@ -40,34 +40,79 @@ class AwesomeStrategy(IStrategy):
!!! Note
If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
***
## Dataframe access
### Storing custom information using DatetimeIndex from `dataframe`
You may access dataframe in various strategy functions by querying it from dataprovider.
Imagine you need to store an indicator like `ATR` or `RSI` into `custom_info`. To use this in a meaningful way, you will not only need the raw data of the indicator, but probably also need to keep the right timestamps.
``` python
from freqtrade.exchange import timeframe_to_prev_date
```python
import talib.abstract as ta
class AwesomeStrategy(IStrategy):
# Create custom dictionary
custom_info = {}
def confirm_trade_exit(self, pair: str, trade: 'Trade', order_type: str, amount: float,
rate: float, time_in_force: str, sell_reason: str,
current_time: 'datetime', **kwargs) -> bool:
# Obtain pair dataframe.
dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
# using "ATR" here as example
dataframe['atr'] = ta.ATR(dataframe)
if self.dp.runmode.value in ('backtest', 'hyperopt'):
# add indicator mapped to correct DatetimeIndex to custom_info
self.custom_info[metadata['pair']] = dataframe[['date', 'atr']].copy().set_index('date')
return dataframe
# Obtain last available candle. Do not use current_time to look up latest candle, because
# current_time points to curret incomplete candle whose data is not available.
last_candle = dataframe.iloc[-1].squeeze()
# <...>
# In dry/live runs trade open date will not match candle open date therefore it must be
# rounded.
trade_date = timeframe_to_prev_date(self.timeframe, trade.open_date_utc)
# Look up trade candle.
trade_candle = dataframe.loc[dataframe['date'] == trade_date]
# trade_candle may be empty for trades that just opened as it is still incomplete.
if not trade_candle.empty:
trade_candle = trade_candle.squeeze()
# <...>
```
!!! Warning
The data is not persisted after a bot-restart (or config-reload). Also, the amount of data should be kept smallish (no DataFrames and such), otherwise the bot will start to consume a lot of memory and eventually run out of memory and crash.
!!! Warning "Using .iloc[-1]"
You can use `.iloc[-1]` here because `get_analyzed_dataframe()` only returns candles that backtesting is allowed to see.
This will not work in `populate_*` methods, so make sure to not use `.iloc[]` in that area.
Also, this will only work starting with version 2021.5.
***
## Custom sell signal
It is possible to define custom sell signals, indicating that specified position should be sold. This is very useful when we need to customize sell conditions for each individual trade, or if you need the trade profit to take the sell decision.
For example you could implement a 1:2 risk-reward ROI with `custom_sell()`.
Using custom_sell() signals in place of stoplosses though *is not recommended*. It is a inferior method to using `custom_stoploss()` in this regard - which also allows you to keep the stoploss on exchange.
!!! Note
If the data is pair-specific, make sure to use pair as one of the keys in the dictionary.
Returning a `string` or `True` from this method is equal to setting sell signal on a candle at specified time. This method is not called when sell signal is set already, or if sell signals are disabled (`use_sell_signal=False` or `sell_profit_only=True` while profit is below `sell_profit_offset`). `string` max length is 64 characters. Exceeding this limit will cause the message to be truncated to 64 characters.
See `custom_stoploss` examples below on how to access the saved dataframe columns
An example of how we can use different indicators depending on the current profit and also sell trades that were open longer than one day:
``` python
class AwesomeStrategy(IStrategy):
def custom_sell(self, pair: str, trade: 'Trade', current_time: 'datetime', current_rate: float,
current_profit: float, **kwargs):
dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
last_candle = dataframe.iloc[-1].squeeze()
# Above 20% profit, sell when rsi < 80
if current_profit > 0.2:
if last_candle['rsi'] < 80:
return 'rsi_below_80'
# Between 2% and 10%, sell if EMA-long above EMA-short
if 0.02 < current_profit < 0.1:
if last_candle['emalong'] > last_candle['emashort']:
return 'ema_long_below_80'
# Sell any positions at a loss if they are held for more than one day.
if current_profit < 0.0 and (current_time - trade.open_date_utc).days >= 1:
return 'unclog'
```
See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks.
## Custom stoploss
@ -110,7 +155,7 @@ class AwesomeStrategy(IStrategy):
:param current_rate: Rate, calculated based on pricing settings in ask_strategy.
:param current_profit: Current profit (as ratio), calculated based on current_rate.
:param **kwargs: Ensure to keep this here so updates to this won't break your strategy.
:return float: New stoploss value, relative to the currentrate
:return float: New stoploss value, relative to the current rate
"""
return -0.04
```
@ -222,7 +267,6 @@ Instead of continuously trailing behind the current price, this example sets fix
* Once profit is > 25% - set stoploss to 15% above open price.
* Once profit is > 40% - set stoploss to 25% above open price.
``` python
from datetime import datetime
from freqtrade.persistence import Trade
@ -248,56 +292,39 @@ class AwesomeStrategy(IStrategy):
# return maximum stoploss value, keeping current stoploss price unchanged
return 1
```
#### Custom stoploss using an indicator from dataframe example
Imagine you want to use `custom_stoploss()` to use a trailing indicator like e.g. "ATR"
See: "Storing custom information using DatetimeIndex from `dataframe`" example above) on how to store the indicator into `custom_info`
!!! Warning
only use .iat[-1] in live mode, not in backtesting/hyperopt
otherwise you will look into the future
see [Common mistakes when developing strategies](strategy-customization.md#common-mistakes-when-developing-strategies) for more info.
Absolute stoploss value may be derived from indicators stored in dataframe. Example uses parabolic SAR below the price as stoploss.
``` python
from freqtrade.persistence import Trade
from freqtrade.state import RunMode
class AwesomeStrategy(IStrategy):
# ... populate_* methods
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
# <...>
dataframe['sar'] = ta.SAR(dataframe)
use_custom_stoploss = True
def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
current_rate: float, current_profit: float, **kwargs) -> float:
result = 1
if self.custom_info and pair in self.custom_info and trade:
# using current_time directly (like below) will only work in backtesting.
# so check "runmode" to make sure that it's only used in backtesting/hyperopt
if self.dp and self.dp.runmode.value in ('backtest', 'hyperopt'):
relative_sl = self.custom_info[pair].loc[current_time]['atr']
# in live / dry-run, it'll be really the current time
else:
# but we can just use the last entry from an already analyzed dataframe instead
dataframe, last_updated = self.dp.get_analyzed_dataframe(pair=pair,
timeframe=self.timeframe)
# WARNING
# only use .iat[-1] in live mode, not in backtesting/hyperopt
# otherwise you will look into the future
# see: https://www.freqtrade.io/en/latest/strategy-customization/#common-mistakes-when-developing-strategies
relative_sl = dataframe['atr'].iat[-1]
dataframe, _ = self.dp.get_analyzed_dataframe(pair, self.timeframe)
last_candle = dataframe.iloc[-1].squeeze()
if (relative_sl is not None):
# new stoploss relative to current_rate
new_stoploss = (current_rate-relative_sl)/current_rate
# turn into relative negative offset required by `custom_stoploss` return implementation
result = new_stoploss - 1
# Use parabolic sar as absolute stoploss price
stoploss_price = last_candle['sar']
return result
# Convert absolute price to percentage relative to current_rate
if stoploss_price < current_rate:
return (stoploss_price / current_rate) - 1
# return maximum stoploss value, keeping current stoploss price unchanged
return 1
```
See [Dataframe access](#dataframe-access) for more information about dataframe use in strategy callbacks.
---
## Custom order timeout rules

10
docs/strategy-customization.md
View File

@ -159,7 +159,7 @@ Edit the method `populate_buy_trend()` in your strategy file to update your buy
It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected.
This will method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action".
This method will also define a new column, `"buy"`, which needs to contain 1 for buys, and 0 for "no action".
Sample from `user_data/strategies/sample_strategy.py`:
@ -193,7 +193,7 @@ Please note that the sell-signal is only used if `use_sell_signal` is set to tru
It's important to always return the dataframe without removing/modifying the columns `"open", "high", "low", "close", "volume"`, otherwise these fields would contain something unexpected.
This will method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action".
This method will also define a new column, `"sell"`, which needs to contain 1 for sells, and 0 for "no action".
Sample from `user_data/strategies/sample_strategy.py`:
@ -422,10 +422,6 @@ if self.dp:
Returns an empty dataframe if the requested pair was not cached.
This should not happen when using whitelisted pairs.
!!! Warning "Warning about backtesting"
This method will return an empty dataframe during backtesting.
### *orderbook(pair, maximum)*
``` python
@ -633,7 +629,7 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati
def custom_stoploss(self, pair: str, trade: 'Trade', current_time: datetime,
current_rate: float, current_profit: float, **kwargs) -> float:
# once the profit has risin above 10%, keep the stoploss at 7% above the open price
# once the profit has risen above 10%, keep the stoploss at 7% above the open price
if current_profit > 0.10:
return stoploss_from_open(0.07, current_profit)

14
docs/strategy_analysis_example.md
View File

@ -195,4 +195,18 @@ graph.show(renderer="browser")
```
## Plot average profit per trade as distribution graph
```python
import plotly.figure_factory as ff
hist_data = [trades.profit_ratio]
group_labels = ['profit_ratio'] # name of the dataset
fig = ff.create_distplot(hist_data, group_labels,bin_size=0.01)
fig.show()
```
Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data.

20
docs/telegram-usage.md
View File

@ -82,12 +82,19 @@ Example configuration showing the different settings:
"buy": "silent",
"sell": "on",
"buy_cancel": "silent",
"sell_cancel": "on"
"sell_cancel": "on",
"buy_fill": "off",
"sell_fill": "off"
},
"balance_dust_level": 0.01
},
```
`buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange.
`sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange.
`*_fill` notifications are off by default and must be explicitly enabled.
`balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown.
## Create a custom keyboard (command shortcut buttons)
@ -258,13 +265,12 @@ Note that for this to work, `forcebuy_enable` needs to be set to true.
### /performance
Return the performance of each crypto-currency the bot has sold.
> Performance:
> 1. `RCN/BTC 57.77%`
> 2. `PAY/BTC 56.91%`
> 3. `VIB/BTC 47.07%`
> 4. `SALT/BTC 30.24%`
> 5. `STORJ/BTC 27.24%`
> 1. `RCN/BTC 0.003 BTC (57.77%) (1)`
> 2. `PAY/BTC 0.0012 BTC (56.91%) (1)`
> 3. `VIB/BTC 0.0011 BTC (47.07%) (1)`
> 4. `SALT/BTC 0.0010 BTC (30.24%) (1)`
> 5. `STORJ/BTC 0.0009 BTC (27.24%) (1)`
> ...
### /balance

199
docs/utils.md
View File

@ -253,18 +253,211 @@ optional arguments:
* Example: see exchanges available for the bot:
```
$ freqtrade list-exchanges
Exchanges available for Freqtrade: _1btcxe, acx, allcoin, bequant, bibox, binance, binanceje, binanceus, bitbank, bitfinex, bitfinex2, bitkk, bitlish, bitmart, bittrex, bitz, bleutrade, btcalpha, btcmarkets, btcturk, buda, cex, cobinhood, coinbaseprime, coinbasepro, coinex, cointiger, coss, crex24, digifinex, dsx, dx, ethfinex, fcoin, fcoinjp, gateio, gdax, gemini, hitbtc2, huobipro, huobiru, idex, kkex, kraken, kucoin, kucoin2, kuna, lbank, mandala, mercado, oceanex, okcoincny, okcoinusd, okex, okex3, poloniex, rightbtc, theocean, tidebit, upbit, zb
Exchanges available for Freqtrade:
Exchange name Valid reason
--------------- ------- --------------------------------------------
aax True
ascendex True missing opt: fetchMyTrades
bequant True
bibox True
bigone True
binance True
binanceus True
bitbank True missing opt: fetchTickers
bitcoincom True
bitfinex True
bitforex True missing opt: fetchMyTrades, fetchTickers
bitget True
bithumb True missing opt: fetchMyTrades
bitkk True missing opt: fetchMyTrades
bitmart True
bitmax True missing opt: fetchMyTrades
bitpanda True
bittrex True
bitvavo True
bitz True missing opt: fetchMyTrades
btcalpha True missing opt: fetchTicker, fetchTickers
btcmarkets True missing opt: fetchTickers
buda True missing opt: fetchMyTrades, fetchTickers
bw True missing opt: fetchMyTrades, fetchL2OrderBook
bybit True
bytetrade True
cdax True
cex True missing opt: fetchMyTrades
coinbaseprime True missing opt: fetchTickers
coinbasepro True missing opt: fetchTickers
coinex True
crex24 True
deribit True
digifinex True
equos True missing opt: fetchTicker, fetchTickers
eterbase True
fcoin True missing opt: fetchMyTrades, fetchTickers
fcoinjp True missing opt: fetchMyTrades, fetchTickers
ftx True
gateio True
gemini True
gopax True
hbtc True
hitbtc True
huobijp True
huobipro True
idex True
kraken True
kucoin True
lbank True missing opt: fetchMyTrades
mercado True missing opt: fetchTickers
ndax True missing opt: fetchTickers
novadax True
okcoin True
okex True
probit True
qtrade True
stex True
timex True
upbit True missing opt: fetchMyTrades
vcc True
zb True missing opt: fetchMyTrades
```
!!! Note "missing opt exchanges"
Values with "missing opt:" might need special configuration (e.g. using orderbook if `fetchTickers` is missing) - but should in theory work (although we cannot guarantee they will).
* Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade):
```
$ freqtrade list-exchanges -a
All exchanges supported by the ccxt library: _1btcxe, acx, adara, allcoin, anxpro, bcex, bequant, bibox, bigone, binance, binanceje, binanceus, bit2c, bitbank, bitbay, bitfinex, bitfinex2, bitflyer, bitforex, bithumb, bitkk, bitlish, bitmart, bitmex, bitso, bitstamp, bitstamp1, bittrex, bitz, bl3p, bleutrade, braziliex, btcalpha, btcbox, btcchina, btcmarkets, btctradeim, btctradeua, btcturk, buda, bxinth, cex, chilebit, cobinhood, coinbase, coinbaseprime, coinbasepro, coincheck, coinegg, coinex, coinexchange, coinfalcon, coinfloor, coingi, coinmarketcap, coinmate, coinone, coinspot, cointiger, coolcoin, coss, crex24, crypton, deribit, digifinex, dsx, dx, ethfinex, exmo, exx, fcoin, fcoinjp, flowbtc, foxbit, fybse, gateio, gdax, gemini, hitbtc, hitbtc2, huobipro, huobiru, ice3x, idex, independentreserve, indodax, itbit, kkex, kraken, kucoin, kucoin2, kuna, lakebtc, latoken, lbank, liquid, livecoin, luno, lykke, mandala, mercado, mixcoins, negociecoins, nova, oceanex, okcoincny, okcoinusd, okex, okex3, paymium, poloniex, rightbtc, southxchange, stronghold, surbitcoin, theocean, therock, tidebit, tidex, upbit, vaultoro, vbtc, virwox, xbtce, yobit, zaif, zb
All exchanges supported by the ccxt library:
Exchange name Valid reason
------------------ ------- ---------------------------------------------------------------------------------------
aax True
aofex False missing: fetchOrder
ascendex True missing opt: fetchMyTrades
bequant True
bibox True
bigone True
binance True
binanceus True
bit2c False missing: fetchOrder, fetchOHLCV
bitbank True missing opt: fetchTickers
bitbay False missing: fetchOrder
bitcoincom True
bitfinex True
bitfinex2 False missing: fetchOrder
bitflyer False missing: fetchOrder, fetchOHLCV
bitforex True missing opt: fetchMyTrades, fetchTickers
bitget True
bithumb True missing opt: fetchMyTrades
bitkk True missing opt: fetchMyTrades
bitmart True
bitmax True missing opt: fetchMyTrades
bitmex False Various reasons.
bitpanda True
bitso False missing: fetchOHLCV
bitstamp False Does not provide history. Details in https://github.com/freqtrade/freqtrade/issues/1983
bitstamp1 False missing: fetchOrder, fetchOHLCV
bittrex True
bitvavo True
bitz True missing opt: fetchMyTrades
bl3p False missing: fetchOrder, fetchOHLCV
bleutrade False missing: fetchOrder
braziliex False missing: fetchOHLCV
btcalpha True missing opt: fetchTicker, fetchTickers
btcbox False missing: fetchOHLCV
btcmarkets True missing opt: fetchTickers
btctradeua False missing: fetchOrder, fetchOHLCV
btcturk False missing: fetchOrder
buda True missing opt: fetchMyTrades, fetchTickers
bw True missing opt: fetchMyTrades, fetchL2OrderBook
bybit True
bytetrade True
cdax True
cex True missing opt: fetchMyTrades
chilebit False missing: fetchOrder, fetchOHLCV
coinbase False missing: fetchOrder, cancelOrder, createOrder, fetchOHLCV
coinbaseprime True missing opt: fetchTickers
coinbasepro True missing opt: fetchTickers
coincheck False missing: fetchOrder, fetchOHLCV
coinegg False missing: fetchOHLCV
coinex True
coinfalcon False missing: fetchOHLCV
coinfloor False missing: fetchOrder, fetchOHLCV
coingi False missing: fetchOrder, fetchOHLCV
coinmarketcap False missing: fetchOrder, cancelOrder, createOrder, fetchBalance, fetchOHLCV
coinmate False missing: fetchOHLCV
coinone False missing: fetchOHLCV
coinspot False missing: fetchOrder, cancelOrder, fetchOHLCV
crex24 True
currencycom False missing: fetchOrder
delta False missing: fetchOrder
deribit True
digifinex True
equos True missing opt: fetchTicker, fetchTickers
eterbase True
exmo False missing: fetchOrder
exx False missing: fetchOHLCV
fcoin True missing opt: fetchMyTrades, fetchTickers
fcoinjp True missing opt: fetchMyTrades, fetchTickers
flowbtc False missing: fetchOrder, fetchOHLCV
foxbit False missing: fetchOrder, fetchOHLCV
ftx True
gateio True
gemini True
gopax True
hbtc True
hitbtc True
hollaex False missing: fetchOrder
huobijp True
huobipro True
idex True
independentreserve False missing: fetchOHLCV
indodax False missing: fetchOHLCV
itbit False missing: fetchOHLCV
kraken True
kucoin True
kuna False missing: fetchOHLCV
lakebtc False missing: fetchOrder, fetchOHLCV
latoken False missing: fetchOrder, fetchOHLCV
lbank True missing opt: fetchMyTrades
liquid False missing: fetchOHLCV
luno False missing: fetchOHLCV
lykke False missing: fetchOHLCV
mercado True missing opt: fetchTickers
mixcoins False missing: fetchOrder, fetchOHLCV
ndax True missing opt: fetchTickers
novadax True
oceanex False missing: fetchOHLCV
okcoin True
okex True
paymium False missing: fetchOrder, fetchOHLCV
phemex False Does not provide history.
poloniex False missing: fetchOrder
probit True
qtrade True
rightbtc False missing: fetchOrder
ripio False missing: fetchOHLCV
southxchange False missing: fetchOrder, fetchOHLCV
stex True
surbitcoin False missing: fetchOrder, fetchOHLCV
therock False missing: fetchOHLCV
tidebit False missing: fetchOrder
tidex False missing: fetchOHLCV
timex True
upbit True missing opt: fetchMyTrades
vbtc False missing: fetchOrder, fetchOHLCV
vcc True
wavesexchange False missing: fetchOrder
whitebit False missing: fetchOrder, cancelOrder, createOrder, fetchBalance
xbtce False missing: fetchOrder, fetchOHLCV
xena False missing: fetchOrder
yobit False missing: fetchOHLCV
zaif False missing: fetchOrder, fetchOHLCV
zb True missing opt: fetchMyTrades
```
## List Timeframes
Use the `list-timeframes` subcommand to see the list of timeframes (ticker intervals) available for the exchange.
Use the `list-timeframes` subcommand to see the list of timeframes available for the exchange.
```
usage: freqtrade list-timeframes [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [-1]

46
docs/webhook-config.md
View File

@ -19,6 +19,11 @@ Sample configuration (tested using IFTTT).
"value1": "Cancelling Open Buy Order for {pair}",
"value2": "limit {limit:8f}",
"value3": "{stake_amount:8f} {stake_currency}"
},
"webhookbuyfill": {
"value1": "Buy Order for {pair} filled",
"value2": "at {open_rate:8f}",
"value3": ""
},
"webhooksell": {
"value1": "Selling {pair}",
@ -30,6 +35,11 @@ Sample configuration (tested using IFTTT).
"value2": "limit {limit:8f}",
"value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})"
},
"webhooksellfill": {
"value1": "Sell Order for {pair} filled",
"value2": "at {close_rate:8f}.",
"value3": ""
},
"webhookstatus": {
"value1": "Status: {status}",
"value2": "",
@ -91,6 +101,21 @@ Possible parameters are:
* `order_type`
* `current_rate`
### Webhookbuyfill
The fields in `webhook.webhookbuyfill` are filled when the bot filled a buy order. Parameters are filled using string.format.
Possible parameters are:
* `trade_id`
* `exchange`
* `pair`
* `open_rate`
* `amount`
* `open_date`
* `stake_amount`
* `stake_currency`
* `fiat_currency`
### Webhooksell
The fields in `webhook.webhooksell` are filled when the bot sells a trade. Parameters are filled using string.format.
@ -103,6 +128,27 @@ Possible parameters are:
* `limit`
* `amount`
* `open_rate`
* `profit_amount`
* `profit_ratio`
* `stake_currency`
* `fiat_currency`
* `sell_reason`
* `order_type`
* `open_date`
* `close_date`
### Webhooksellfill
The fields in `webhook.webhooksellfill` are filled when the bot fills a sell order (closes a Trae). Parameters are filled using string.format.
Possible parameters are:
* `trade_id`
* `exchange`
* `pair`
* `gain`
* `close_rate`
* `amount`
* `open_rate`
* `current_rate`
* `profit_amount`
* `profit_ratio`

2
environment.yml
View File

@ -4,7 +4,7 @@ channels:
# - defaults
dependencies:
# 1/4 req main
- python>=3.7
- python>=3.7,<3.9
- numpy
- pandas
- pip

7
freqtrade/commands/arguments.py
View File

@ -17,7 +17,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
"max_open_trades", "stake_amount", "fee"]
"max_open_trades", "stake_amount", "fee", "pairs"]
ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
"enable_protections", "dry_run_wallet",
@ -60,8 +60,9 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange",
"timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
"download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv",
"dataformat_trades"]
ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
"db_url", "trade_source", "export", "exportfilename",

32
freqtrade/commands/build_config_commands.py
View File

@ -1,9 +1,11 @@
import logging
import secrets
from pathlib import Path
from typing import Any, Dict, List
from questionary import Separator, prompt
from freqtrade.configuration.directory_operations import chown_user_directory
from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges
@ -138,6 +140,32 @@ def ask_user_config() -> Dict[str, Any]:
"message": "Insert Telegram chat id",
"when": lambda x: x['telegram']
},
{
"type": "confirm",
"name": "api_server",
"message": "Do you want to enable the Rest API (includes FreqUI)?",
"default": False,
},
{
"type": "text",
"name": "api_server_listen_addr",
"message": "Insert Api server Listen Address (best left untouched default!)",
"default": "127.0.0.1",
"when": lambda x: x['api_server']
},
{
"type": "text",
"name": "api_server_username",
"message": "Insert api-server username",
"default": "freqtrader",
"when": lambda x: x['api_server']
},
{
"type": "text",
"name": "api_server_password",
"message": "Insert api-server password",
"when": lambda x: x['api_server']
},
]
answers = prompt(questions)
@ -145,6 +173,9 @@ def ask_user_config() -> Dict[str, Any]:
# Interrupted questionary sessions return an empty dict.
raise OperationalException("User interrupted interactive questions.")
# Force JWT token to be a random string
answers['api_server_jwt_key'] = secrets.token_hex()
return answers
@ -186,6 +217,7 @@ def start_new_config(args: Dict[str, Any]) -> None:
"""
config_path = Path(args['config'][0])
chown_user_directory(config_path.parent)
if config_path.exists():
overwrite = ask_user_overwrite(config_path)
if overwrite:

13
freqtrade/commands/cli_options.py
View File

@ -118,7 +118,7 @@ AVAILABLE_CLI_OPTIONS = {
# Optimize common
"timeframe": Arg(
'-i', '--timeframe', '--ticker-interval',
help='Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`).',
help='Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).',
),
"timerange": Arg(
'--timerange',
@ -195,6 +195,7 @@ AVAILABLE_CLI_OPTIONS = {
'--hyperopt',
help='Specify hyperopt class name which will be used by the bot.',
metavar='NAME',
required=False,
),
"hyperopt_path": Arg(
'--hyperopt-path',
@ -266,7 +267,7 @@ AVAILABLE_CLI_OPTIONS = {
default=1,
),
"hyperopt_loss": Arg(
'--hyperopt-loss',
'--hyperopt-loss', '--hyperoptloss',
help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). '
'Different functions can generate completely different results, '
'since the target for optimization is different. Built-in Hyperopt-loss-functions are: '
@ -329,7 +330,7 @@ AVAILABLE_CLI_OPTIONS = {
# Script options
"pairs": Arg(
'-p', '--pairs',
help='Show profits for only these pairs. Pairs are space-separated.',
help='Limit command to these pairs. Pairs are space-separated.',
nargs='+',
),
# Download data
@ -344,6 +345,12 @@ AVAILABLE_CLI_OPTIONS = {
type=check_int_positive,
metavar='INT',
),
"new_pairs_days": Arg(
'--new-pairs-days',
help='Download data of new pairs for given number of days. Default: `%(default)s`.',
type=check_int_positive,
metavar='INT',
),
"download_trades": Arg(
'--dl-trades',
help='Download trades instead of OHLCV data. The bot will resample trades to the '

9
freqtrade/commands/data_commands.py
View File

@ -62,8 +62,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
if config.get('download_trades'):
pairs_not_available = refresh_backtest_trades_data(
exchange, pairs=expanded_pairs, datadir=config['datadir'],
timerange=timerange, erase=bool(config.get('erase')),
data_format=config['dataformat_trades'])
timerange=timerange, new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
@ -75,8 +75,9 @@ def start_download_data(args: Dict[str, Any]) -> None:
else:
pairs_not_available = refresh_backtest_ohlcv_data(
exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format=config['dataformat_ohlcv'])
datadir=config['datadir'], timerange=timerange,
new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'])
except KeyboardInterrupt:
sys.exit("SIGINT received, aborting ...")

78
freqtrade/commands/hyperopt_commands.py
View File

@ -7,6 +7,7 @@ from colorama import init as colorama_init
from freqtrade.configuration import setup_utils_configuration
from freqtrade.data.btanalysis import get_latest_hyperopt_file
from freqtrade.exceptions import OperationalException
from freqtrade.optimize.optimize_reports import show_backtest_result
from freqtrade.state import RunMode
@ -125,6 +126,12 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
if epochs:
val = epochs[n]
metrics = val['results_metrics']
if 'strategy_name' in metrics:
show_backtest_result(metrics['strategy_name'], metrics,
metrics['stake_currency'])
HyperoptTools.print_epoch_details(val, total_epochs, print_json, no_header,
header_str="Epoch details")
@ -132,11 +139,13 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None:
def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
"""
Filter our items from the list of hyperopt results
TODO: after 2021.5 remove all "legacy" mode queries.
"""
if filteroptions['only_best']:
epochs = [x for x in epochs if x['is_best']]
if filteroptions['only_profitable']:
epochs = [x for x in epochs if x['results_metrics']['profit'] > 0]
epochs = [x for x in epochs if x['results_metrics'].get(
'profit', x['results_metrics'].get('profit_total', 0)) > 0]
epochs = _hyperopt_filter_epochs_trade_count(epochs, filteroptions)
@ -153,34 +162,55 @@ def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List:
return epochs
def _hyperopt_filter_epochs_trade(epochs: List, trade_count: int):
"""
Filter epochs with trade-counts > trades
"""
return [
x for x in epochs
if x['results_metrics'].get(
'trade_count', x['results_metrics'].get('total_trades', 0)
) > trade_count
]
def _hyperopt_filter_epochs_trade_count(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_trades'] > 0:
epochs = [
x for x in epochs
if x['results_metrics']['trade_count'] > filteroptions['filter_min_trades']
]
epochs = _hyperopt_filter_epochs_trade(epochs, filteroptions['filter_min_trades'])
if filteroptions['filter_max_trades'] > 0:
epochs = [
x for x in epochs
if x['results_metrics']['trade_count'] < filteroptions['filter_max_trades']
if x['results_metrics'].get(
'trade_count', x['results_metrics'].get('total_trades')
) < filteroptions['filter_max_trades']
]
return epochs
def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
def get_duration_value(x):
# Duration in minutes ...
if 'duration' in x['results_metrics']:
return x['results_metrics']['duration']
else:
# New mode
avg = x['results_metrics']['holding_avg']
return avg.total_seconds() // 60
if filteroptions['filter_min_avg_time'] is not None:
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics']['duration'] > filteroptions['filter_min_avg_time']
if get_duration_value(x) > filteroptions['filter_min_avg_time']
]
if filteroptions['filter_max_avg_time'] is not None:
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics']['duration'] < filteroptions['filter_max_avg_time']
if get_duration_value(x) < filteroptions['filter_max_avg_time']
]
return epochs
@ -189,28 +219,36 @@ def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List:
def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_avg_profit'] is not None:
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics']['avg_profit'] > filteroptions['filter_min_avg_profit']
if x['results_metrics'].get(
'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
) > filteroptions['filter_min_avg_profit']
]
if filteroptions['filter_max_avg_profit'] is not None:
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics']['avg_profit'] < filteroptions['filter_max_avg_profit']
if x['results_metrics'].get(
'avg_profit', x['results_metrics'].get('profit_mean', 0) * 100
) < filteroptions['filter_max_avg_profit']
]
if filteroptions['filter_min_total_profit'] is not None:
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics']['profit'] > filteroptions['filter_min_total_profit']
if x['results_metrics'].get(
'profit', x['results_metrics'].get('profit_total_abs', 0)
) > filteroptions['filter_min_total_profit']
]
if filteroptions['filter_max_total_profit'] is not None:
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [
x for x in epochs
if x['results_metrics']['profit'] < filteroptions['filter_max_total_profit']
if x['results_metrics'].get(
'profit', x['results_metrics'].get('profit_total_abs', 0)
) < filteroptions['filter_max_total_profit']
]
return epochs
@ -218,11 +256,11 @@ def _hyperopt_filter_epochs_profit(epochs: List, filteroptions: dict) -> List:
def _hyperopt_filter_epochs_objective(epochs: List, filteroptions: dict) -> List:
if filteroptions['filter_min_objective'] is not None:
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] < filteroptions['filter_min_objective']]
if filteroptions['filter_max_objective'] is not None:
epochs = [x for x in epochs if x['results_metrics']['trade_count'] > 0]
epochs = _hyperopt_filter_epochs_trade(epochs, 0)
epochs = [x for x in epochs if x['loss'] > filteroptions['filter_max_objective']]

18
freqtrade/commands/list_commands.py
View File

@ -13,7 +13,7 @@ from tabulate import tabulate
from freqtrade.configuration import setup_utils_configuration
from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import available_exchanges, ccxt_exchanges, market_is_active
from freqtrade.exchange import market_is_active, validate_exchanges
from freqtrade.misc import plural
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.state import RunMode
@ -28,14 +28,18 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
:param args: Cli args from Arguments()
:return: None
"""
exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges()
exchanges = validate_exchanges(args['list_exchanges_all'])
if args['print_one_column']:
print('\n'.join(exchanges))
print('\n'.join([e[0] for e in exchanges]))
else:
if args['list_exchanges_all']:
print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}")
print("All exchanges supported by the ccxt library:")
else:
print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}")
print("Exchanges available for Freqtrade:")
exchanges = [e for e in exchanges if e[1] is not False]
print(tabulate(exchanges, headers=['Exchange name', 'Valid', 'reason']))
def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
@ -99,7 +103,7 @@ def start_list_hyperopts(args: Dict[str, Any]) -> None:
def start_list_timeframes(args: Dict[str, Any]) -> None:
"""
Print ticker intervals (timeframes) available on Exchange
Print timeframes available on Exchange
"""
config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
# Do not use timeframe set in the config
@ -177,7 +181,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
# human-readable formats.
print()
if len(pairs):
if pairs:
if args.get('print_list', False):
# print data as a list, with human-readable summary
print(f"{summary_str}: {', '.join(pairs.keys())}.")

14
freqtrade/configuration/check_exchange.py
View File

@ -2,8 +2,8 @@ import logging
from typing import Any, Dict
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, is_exchange_bad,
is_exchange_known_ccxt, is_exchange_officially_supported)
from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
is_exchange_officially_supported, validate_exchange)
from freqtrade.state import RunMode
@ -57,9 +57,13 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
f'{", ".join(available_exchanges())}'
)
if check_for_bad and is_exchange_bad(exchange):
raise OperationalException(f'Exchange "{exchange}" is known to not work with the bot yet. '
f'Reason: {get_exchange_bad_reason(exchange)}')
valid, reason = validate_exchange(exchange)
if not valid:
if check_for_bad:
raise OperationalException(f'Exchange "{exchange}" will not work with Freqtrade. '
f'Reason: {reason}')
else:
logger.warning(f'Exchange "{exchange}" will not work with Freqtrade. Reason: {reason}')
if is_exchange_officially_supported(exchange):
logger.info(f'Exchange "{exchange}" is officially supported '

5
freqtrade/configuration/config_validation.py
View File

@ -149,11 +149,6 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
if not conf.get('edge', {}).get('enabled'):
return
if conf.get('pairlist', {}).get('method') == 'VolumePairList':
raise OperationalException(
"Edge and VolumePairList are incompatible, "
"Edge will override whatever pairs VolumePairlist selects."
)
if not conf.get('ask_strategy', {}).get('use_sell_signal', True):
raise OperationalException(
"Edge requires `use_sell_signal` to be True, otherwise no sells will happen."

22
freqtrade/configuration/configuration.py
View File

@ -11,10 +11,10 @@ from freqtrade import constants
from freqtrade.configuration.check_exchange import check_exchange
from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
from freqtrade.configuration.load_config import load_config_file
from freqtrade.configuration.load_config import load_config_file, load_file
from freqtrade.exceptions import OperationalException
from freqtrade.loggers import setup_logging
from freqtrade.misc import deep_merge_dicts, json_load
from freqtrade.misc import deep_merge_dicts
from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode
@ -75,8 +75,6 @@ class Configuration:
# Normalize config
if 'internals' not in config:
config['internals'] = {}
# TODO: This can be deleted along with removal of deprecated
# experimental settings
if 'ask_strategy' not in config:
config['ask_strategy'] = {}
@ -108,6 +106,8 @@ class Configuration:
self._process_plot_options(config)
self._process_data_options(config)
# Check if the exchange set by the user is supported
check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
@ -399,6 +399,11 @@ class Configuration:
self._args_to_config(config, argname='dataformat_trades',
logstring='Using "{}" to store trades data.')
def _process_data_options(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='new_pairs_days',
logstring='Detected --new-pairs-days: {}')
def _process_runmode(self, config: Dict[str, Any]) -> None:
self._args_to_config(config, argname='dry_run',
@ -445,6 +450,7 @@ class Configuration:
"""
if "pairs" in config:
config['exchange']['pair_whitelist'] = config['pairs']
return
if "pairs_file" in self.args and self.args["pairs_file"]:
@ -454,9 +460,8 @@ class Configuration:
# or if pairs file is specified explicitely
if not pairs_file.exists():
raise OperationalException(f'No pairs file found with path "{pairs_file}".')
with pairs_file.open('r') as f:
config['pairs'] = json_load(f)
config['pairs'].sort()
config['pairs'] = load_file(pairs_file)
config['pairs'].sort()
return
if 'config' in self.args and self.args['config']:
@ -466,7 +471,6 @@ class Configuration:
# Fall back to /dl_path/pairs.json
pairs_file = config['datadir'] / 'pairs.json'
if pairs_file.exists():
with pairs_file.open('r') as f:
config['pairs'] = json_load(f)
config['pairs'] = load_file(pairs_file)
if 'pairs' in config:
config['pairs'].sort()

19
freqtrade/configuration/directory_operations.py
View File

@ -24,6 +24,21 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Pat
return folder
def chown_user_directory(directory: Path) -> None:
"""
Use Sudo to change permissions of the home-directory if necessary
Only applies when running in docker!
"""
import os
if os.environ.get('FT_APP_ENV') == 'docker':
try:
import subprocess
subprocess.check_output(
['sudo', 'chown', '-R', 'ftuser:', str(directory.resolve())])
except Exception:
logger.warning(f"Could not chown {directory}")
def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
"""
Create userdata directory structure.
@ -37,6 +52,7 @@ def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "logs",
"notebooks", "plot", "strategies", ]
folder = Path(directory)
chown_user_directory(folder)
if not folder.is_dir():
if create_dir:
folder.mkdir(parents=True)
@ -72,6 +88,5 @@ def copy_sample_files(directory: Path, overwrite: bool = False) -> None:
if not overwrite:
logger.warning(f"File `{targetfile}` exists already, not deploying sample file.")
continue
else:
logger.warning(f"File `{targetfile}` exists already, overwriting.")
logger.warning(f"File `{targetfile}` exists already, overwriting.")
shutil.copy(str(sourcedir / source), str(targetfile))

9
freqtrade/configuration/load_config.py
View File

@ -38,6 +38,15 @@ def log_config_error_range(path: str, errmsg: str) -> str:
return ''
def load_file(path: Path) -> Dict[str, Any]:
try:
with path.open('r') as file:
config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
except FileNotFoundError:
raise OperationalException(f'File file "{path}" not found!')
return config
def load_config_file(path: str) -> Dict[str, Any]:
"""
Loads a config file from the given path

7
freqtrade/configuration/timerange.py
View File

@ -3,6 +3,7 @@ This module contains the argument manager class
"""
import logging
import re
from datetime import datetime
from typing import Optional
import arrow
@ -43,7 +44,7 @@ class TimeRange:
self.startts = self.startts - seconds
def adjust_start_if_necessary(self, timeframe_secs: int, startup_candles: int,
min_date: arrow.Arrow) -> None:
min_date: datetime) -> None:
"""
Adjust startts by <startup_candles> candles.
Applies only if no startup-candles have been available.
@ -54,11 +55,11 @@ class TimeRange:
:return: None (Modifies the object in place)
"""
if (not self.starttype or (startup_candles
and min_date.int_timestamp >= self.startts)):
and min_date.timestamp() >= self.startts)):
# If no startts was defined, or backtest-data starts at the defined backtest-date
logger.warning("Moving start-date by %s candles to account for startup time.",
startup_candles)
self.startts = (min_date.int_timestamp + timeframe_secs * startup_candles)
self.startts = int(min_date.timestamp() + timeframe_secs * startup_candles)
self.starttype = 'date'
@staticmethod

23
freqtrade/constants.py
View File

@ -11,6 +11,7 @@ DEFAULT_EXCHANGE = 'bittrex'
PROCESS_THROTTLE_SECS = 5 # sec
HYPEROPT_EPOCH = 100 # epochs
RETRY_TIMEOUT = 30 # sec
TIMEOUT_UNITS = ['minutes', 'seconds']
DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite'
DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite'
UNLIMITED_STAKE_AMOUNT = 'unlimited'
@ -26,7 +27,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
'AgeFilter', 'PerformanceFilter', 'PrecisionFilter',
'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter',
'SpreadFilter']
'SpreadFilter', 'VolatilityFilter']
AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
DRY_RUN_WALLET = 1000
@ -96,6 +97,7 @@ CONF_SCHEMA = {
'type': 'object',
'properties': {
'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1},
'new_pairs_days': {'type': 'integer', 'default': 30},
'timeframe': {'type': 'string'},
'stake_currency': {'type': 'string'},
'stake_amount': {
@ -136,7 +138,8 @@ CONF_SCHEMA = {
'type': 'object',
'properties': {
'buy': {'type': 'number', 'minimum': 1},
'sell': {'type': 'number', 'minimum': 1}
'sell': {'type': 'number', 'minimum': 1},
'unit': {'type': 'string', 'enum': TIMEOUT_UNITS, 'default': 'minutes'}
}
},
'bid_strategy': {
@ -176,7 +179,7 @@ CONF_SCHEMA = {
'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50},
'use_sell_signal': {'type': 'boolean'},
'sell_profit_only': {'type': 'boolean'},
'sell_profit_offset': {'type': 'number', 'minimum': 0.0},
'sell_profit_offset': {'type': 'number'},
'ignore_roi_if_buy_signal': {'type': 'boolean'}
}
},
@ -246,14 +249,24 @@ CONF_SCHEMA = {
'balance_dust_level': {'type': 'number', 'minimum': 0.0},
'notification_settings': {
'type': 'object',
'default': {},
'properties': {
'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}
'buy_fill': {'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS,
'default': 'off'
},
'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
'sell_fill': {
'type': 'string',
'enum': TELEGRAM_SETTING_OPTIONS,
'default': 'off'
},
}
}
},

56
freqtrade/data/btanalysis.py
View File

@ -156,33 +156,35 @@ def load_backtest_data(filename: Union[Path, str], strategy: Optional[str] = Non
data = data['strategy'][strategy]['trades']
df = pd.DataFrame(data)
df['open_date'] = pd.to_datetime(df['open_date'],
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
utc=True,
infer_datetime_format=True
)
if not df.empty:
df['open_date'] = pd.to_datetime(df['open_date'],
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
utc=True,
infer_datetime_format=True
)
else:
# old format - only with lists.
df = pd.DataFrame(data, columns=BT_DATA_COLUMNS_OLD)
df['open_date'] = pd.to_datetime(df['open_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
# Create compatibility with new format
df['profit_abs'] = df['close_rate'] - df['open_rate']
if 'profit_ratio' not in df.columns:
df['profit_ratio'] = df['profit_percent']
df = df.sort_values("open_date").reset_index(drop=True)
if not df.empty:
df['open_date'] = pd.to_datetime(df['open_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
df['close_date'] = pd.to_datetime(df['close_date'],
unit='s',
utc=True,
infer_datetime_format=True
)
# Create compatibility with new format
df['profit_abs'] = df['close_rate'] - df['open_rate']
if not df.empty:
if 'profit_ratio' not in df.columns:
df['profit_ratio'] = df['profit_percent']
df = df.sort_values("open_date").reset_index(drop=True)
return df
@ -337,7 +339,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
"""
Adds a column `col_name` with the cumulative profit for the given trades array.
:param df: DataFrame with date index
:param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
:param trades: DataFrame containing trades (requires columns close_date and profit_abs)
:param col_name: Column name that will be assigned the results
:param timeframe: Timeframe used during the operations
:return: Returns df with one additional column, col_name, containing the cumulative profit.
@ -349,8 +351,8 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
timeframe_minutes = timeframe_to_minutes(timeframe)
# Resample to timeframe to make sure trades match candles
_trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
)[['profit_ratio']].sum()
df.loc[:, col_name] = _trades_sum['profit_ratio'].cumsum()
)[['profit_abs']].sum()
df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
# Set first value to 0
df.loc[df.iloc[0].name, col_name] = 0
# FFill to get continuous

46
freqtrade/data/converter.py
View File

@ -110,28 +110,62 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
df.reset_index(inplace=True)
len_before = len(dataframe)
len_after = len(df)
pct_missing = (len_after - len_before) / len_before if len_before > 0 else 0
if len_before != len_after:
logger.info(f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}")
message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}"
f" - {round(pct_missing * 100, 2)}%")
if pct_missing > 0.01:
logger.info(message)
else:
# Don't be verbose if only a small amount is missing
logger.debug(message)
return df
def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date') -> DataFrame:
def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
startup_candles: int = 0) -> DataFrame:
"""
Trim dataframe based on given timerange
:param df: Dataframe to trim
:param timerange: timerange (use start and end date if available)
:param: df_date_col: Column in the dataframe to use as Date column
:param df_date_col: Column in the dataframe to use as Date column
:param startup_candles: When not 0, is used instead the timerange start date
:return: trimmed dataframe
"""
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
df = df.loc[df[df_date_col] >= start, :]
if startup_candles:
# Trim candles instead of timeframe in case of given startup_candle count
df = df.iloc[startup_candles:, :]
else:
if timerange.starttype == 'date':
start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
df = df.loc[df[df_date_col] >= start, :]
if timerange.stoptype == 'date':
stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
df = df.loc[df[df_date_col] <= stop, :]
return df
def trim_dataframes(preprocessed: Dict[str, DataFrame], timerange,
startup_candles: int) -> Dict[str, DataFrame]:
"""
Trim startup period from analyzed dataframes
:param preprocessed: Dict of pair: dataframe
:param timerange: timerange (use start and end date if available)
:param startup_candles: Startup-candles that should be removed
:return: Dict of trimmed dataframes
"""
processed: Dict[str, DataFrame] = {}
for pair, df in preprocessed.items():
trimed_df = trim_dataframe(df, timerange, startup_candles=startup_candles)
if not trimed_df.empty:
processed[pair] = trimed_df
else:
logger.warning(f'{pair} has no data left after adjusting for startup candles, '
f'skipping.')
return processed
def order_book_to_dataframe(bids: list, asks: list) -> DataFrame:
"""
TODO: This should get a dedicated test

176
freqtrade/data/dataprovider.py
View File

@ -19,14 +19,25 @@ from freqtrade.state import RunMode
logger = logging.getLogger(__name__)
NO_EXCHANGE_EXCEPTION = 'Exchange is not available to DataProvider.'
MAX_DATAFRAME_CANDLES = 1000
class DataProvider:
def __init__(self, config: dict, exchange: Exchange, pairlists=None) -> None:
def __init__(self, config: dict, exchange: Optional[Exchange], pairlists=None) -> None:
self._config = config
self._exchange = exchange
self._pairlists = pairlists
self.__cached_pairs: Dict[PairWithTimeframe, Tuple[DataFrame, datetime]] = {}
self.__slice_index: Optional[int] = None
def _set_dataframe_max_index(self, limit_index: int):
"""
Limit analyzed dataframe to max specified index.
:param limit_index: dataframe index.
"""
self.__slice_index = limit_index
def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None:
"""
@ -45,40 +56,6 @@ class DataProvider:
"""
self._pairlists = pairlists
def refresh(self,
pairlist: ListPairsWithTimeframes,
helping_pairs: ListPairsWithTimeframes = None) -> None:
"""
Refresh data, called with each cycle
"""
if helping_pairs:
self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
else:
self._exchange.refresh_latest_ohlcv(pairlist)
@property
def available_pairs(self) -> ListPairsWithTimeframes:
"""
Return a list of tuples containing (pair, timeframe) for which data is currently cached.
Should be whitelist + open trades.
"""
return list(self._exchange._klines.keys())
def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
"""
Get candle (OHLCV) data for the given pair as DataFrame
Please use the `available_pairs` method to verify which pairs are currently cached.
:param pair: pair to get the data for
:param timeframe: Timeframe to get data for
:param copy: copy dataframe before returning if True.
Use False only for read-only operations (where the dataframe is not modified)
"""
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
return self._exchange.klines((pair, timeframe or self._config['timeframe']),
copy=copy)
else:
return DataFrame()
def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame:
"""
Get stored historical candle (OHLCV) data
@ -111,47 +88,27 @@ class DataProvider:
def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]:
"""
Retrieve the analyzed dataframe. Returns the full dataframe in trade mode (live / dry),
and the last 1000 candles (up to the time evaluated at this moment) in all other modes.
:param pair: pair to get the data for
:param timeframe: timeframe to get data for
:return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe
combination.
Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached.
"""
if (pair, timeframe) in self.__cached_pairs:
return self.__cached_pairs[(pair, timeframe)]
pair_key = (pair, timeframe)
if pair_key in self.__cached_pairs:
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
df, date = self.__cached_pairs[pair_key]
else:
df, date = self.__cached_pairs[pair_key]
if self.__slice_index is not None:
max_index = self.__slice_index
df = df.iloc[max(0, max_index - MAX_DATAFRAME_CANDLES):max_index]
return df, date
else:
return (DataFrame(), datetime.fromtimestamp(0, tz=timezone.utc))
def market(self, pair: str) -> Optional[Dict[str, Any]]:
"""
Return market data for the pair
:param pair: Pair to get the data for
:return: Market data dict from ccxt or None if market info is not available for the pair
"""
return self._exchange.markets.get(pair)
def ticker(self, pair: str):
"""
Return last ticker data from exchange
:param pair: Pair to get the data for
:return: Ticker dict from exchange or empty dict if ticker is not available for the pair
"""
try:
return self._exchange.fetch_ticker(pair)
except ExchangeError:
return {}
def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
"""
Fetch latest l2 orderbook data
Warning: Does a network request - so use with common sense.
:param pair: pair to get the data for
:param maximum: Maximum number of orderbook entries to query
:return: dict including bids/asks with a total of `maximum` entries.
"""
return self._exchange.fetch_l2_order_book(pair, maximum)
@property
def runmode(self) -> RunMode:
"""
@ -170,6 +127,89 @@ class DataProvider:
"""
if self._pairlists:
return self._pairlists.whitelist
return self._pairlists.whitelist.copy()
else:
raise OperationalException("Dataprovider was not initialized with a pairlist provider.")
def clear_cache(self):
"""
Clear pair dataframe cache.
"""
self.__cached_pairs = {}
# Exchange functions
def refresh(self,
pairlist: ListPairsWithTimeframes,
helping_pairs: ListPairsWithTimeframes = None) -> None:
"""
Refresh data, called with each cycle
"""
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
if helping_pairs:
self._exchange.refresh_latest_ohlcv(pairlist + helping_pairs)
else:
self._exchange.refresh_latest_ohlcv(pairlist)
@property
def available_pairs(self) -> ListPairsWithTimeframes:
"""
Return a list of tuples containing (pair, timeframe) for which data is currently cached.
Should be whitelist + open trades.
"""
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
return list(self._exchange._klines.keys())
def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame:
"""
Get candle (OHLCV) data for the given pair as DataFrame
Please use the `available_pairs` method to verify which pairs are currently cached.
:param pair: pair to get the data for
:param timeframe: Timeframe to get data for
:param copy: copy dataframe before returning if True.
Use False only for read-only operations (where the dataframe is not modified)
"""
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE):
return self._exchange.klines((pair, timeframe or self._config['timeframe']),
copy=copy)
else:
return DataFrame()
def market(self, pair: str) -> Optional[Dict[str, Any]]:
"""
Return market data for the pair
:param pair: Pair to get the data for
:return: Market data dict from ccxt or None if market info is not available for the pair
"""
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
return self._exchange.markets.get(pair)
def ticker(self, pair: str):
"""
Return last ticker data from exchange
:param pair: Pair to get the data for
:return: Ticker dict from exchange or empty dict if ticker is not available for the pair
"""
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
try:
return self._exchange.fetch_ticker(pair)
except ExchangeError:
return {}
def orderbook(self, pair: str, maximum: int) -> Dict[str, List]:
"""
Fetch latest l2 orderbook data
Warning: Does a network request - so use with common sense.
:param pair: pair to get the data for
:param maximum: Maximum number of orderbook entries to query
:return: dict including bids/asks with a total of `maximum` entries.
"""
if self._exchange is None:
raise OperationalException(NO_EXCHANGE_EXCEPTION)
return self._exchange.fetch_l2_order_book(pair, maximum)

2
freqtrade/data/history/hdf5datahandler.py
View File

@ -89,7 +89,7 @@ class HDF5DataHandler(IDataHandler):
if timerange.starttype == 'date':
where.append(f"date >= Timestamp({timerange.startts * 1e9})")
if timerange.stoptype == 'date':
where.append(f"date < Timestamp({timerange.stopts * 1e9})")
where.append(f"date <= Timestamp({timerange.stopts * 1e9})")
pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)

28
freqtrade/data/history/history_utils.py
View File

@ -155,6 +155,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
def _download_pair_history(datadir: Path,
exchange: Exchange,
pair: str, *,
new_pairs_days: int = 30,
timeframe: str = '5m',
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler = None) -> bool:
@ -193,7 +194,7 @@ def _download_pair_history(datadir: Path,
timeframe=timeframe,
since_ms=since_ms if since_ms else
int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000
days=-new_pairs_days).float_timestamp) * 1000
)
# TODO: Maybe move parsing to exchange class (?)
new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@ -223,7 +224,8 @@ def _download_pair_history(datadir: Path,
def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
datadir: Path, timerange: Optional[TimeRange] = None,
erase: bool = False, data_format: str = None) -> List[str]:
new_pairs_days: int = 30, erase: bool = False,
data_format: str = None) -> List[str]:
"""
Refresh stored ohlcv data for backtesting and hyperopt operations.
Used by freqtrade download-data subcommand.
@ -246,12 +248,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
logger.info(f'Downloading pair {pair}, interval {timeframe}.')
_download_pair_history(datadir=datadir, exchange=exchange,
pair=pair, timeframe=str(timeframe),
new_pairs_days=new_pairs_days,
timerange=timerange, data_handler=data_handler)
return pairs_not_available
def _download_trades_history(exchange: Exchange,
pair: str, *,
new_pairs_days: int = 30,
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler
) -> bool:
@ -261,9 +265,13 @@ def _download_trades_history(exchange: Exchange,
"""
try:
since = timerange.startts * 1000 if \
(timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
days=-30).float_timestamp) * 1000
until = None
if (timerange and timerange.starttype == 'date'):
since = timerange.startts * 1000
if timerange.stoptype == 'date':
until = timerange.stopts * 1000
else:
since = int(arrow.utcnow().shift(days=-new_pairs_days).float_timestamp) * 1000
trades = data_handler.trades_load(pair)
@ -291,6 +299,7 @@ def _download_trades_history(exchange: Exchange,
# Default since_ms to 30 days if nothing is given
new_trades = exchange.get_historic_trades(pair=pair,
since=since,
until=until,
from_id=from_id,
)
trades.extend(new_trades[1])
@ -311,8 +320,8 @@ def _download_trades_history(exchange: Exchange,
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
timerange: TimeRange, erase: bool = False,
data_format: str = 'jsongz') -> List[str]:
timerange: TimeRange, new_pairs_days: int = 30,
erase: bool = False, data_format: str = 'jsongz') -> List[str]:
"""
Refresh stored trades data for backtesting and hyperopt operations.
Used by freqtrade download-data subcommand.
@ -333,6 +342,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
logger.info(f'Downloading trades for pair {pair}.')
_download_trades_history(exchange=exchange,
pair=pair,
new_pairs_days=new_pairs_days,
timerange=timerange,
data_handler=data_handler)
return pairs_not_available
@ -362,7 +372,7 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str],
logger.exception(f'Could not convert {pair} to OHLCV.')
def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]:
def get_timerange(data: Dict[str, DataFrame]) -> Tuple[datetime, datetime]:
"""
Get the maximum common timerange for the given backtest data.
@ -370,7 +380,7 @@ def get_timerange(data: Dict[str, DataFrame]) -> Tuple[arrow.Arrow, arrow.Arrow]
:return: tuple containing min_date, max_date
"""
timeranges = [
(arrow.get(frame['date'].min()), arrow.get(frame['date'].max()))
(frame['date'].min().to_pydatetime(), frame['date'].max().to_pydatetime())
for frame in data.values()
]
return (min(timeranges, key=operator.itemgetter(0))[0],

52
freqtrade/edge/edge_positioning.py
View File

@ -1,6 +1,8 @@
# pragma pylint: disable=W0603
""" Edge positioning package """
import logging
from collections import defaultdict
from copy import deepcopy
from typing import Any, Dict, List, NamedTuple
import arrow
@ -12,8 +14,10 @@ from freqtrade.configuration import TimeRange
from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT
from freqtrade.data.history import get_timerange, load_data, refresh_data
from freqtrade.exceptions import OperationalException
from freqtrade.exchange.exchange import timeframe_to_seconds
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.strategy.interface import SellType
from freqtrade.state import RunMode
from freqtrade.strategy.interface import IStrategy, SellType
logger = logging.getLogger(__name__)
@ -45,7 +49,7 @@ class Edge:
self.config = config
self.exchange = exchange
self.strategy = strategy
self.strategy: IStrategy = strategy
self.edge_config = self.config.get('edge', {})
self._cached_pairs: Dict[str, Any] = {} # Keeps a list of pairs
@ -81,12 +85,16 @@ class Edge:
if config.get('fee'):
self.fee = config['fee']
else:
self.fee = self.exchange.get_fee(symbol=expand_pairlist(
self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
try:
self.fee = self.exchange.get_fee(symbol=expand_pairlist(
self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
except IndexError:
self.fee = None
def calculate(self, pairs: List[str]) -> bool:
if self.fee is None and pairs:
self.fee = self.exchange.get_fee(pairs[0])
def calculate(self) -> bool:
pairs = expand_pairlist(self.config['exchange']['pair_whitelist'],
list(self.exchange.markets))
heartbeat = self.edge_config.get('process_throttle_secs')
if (self._last_updated > 0) and (
@ -98,14 +106,33 @@ class Edge:
logger.info('Using local backtesting data (using whitelist in given config) ...')
if self._refresh_pairs:
timerange_startup = deepcopy(self._timerange)
timerange_startup.subtract_start(timeframe_to_seconds(
self.strategy.timeframe) * self.strategy.startup_candle_count)
refresh_data(
datadir=self.config['datadir'],
pairs=pairs,
exchange=self.exchange,
timeframe=self.strategy.timeframe,
timerange=self._timerange,
timerange=timerange_startup,
data_format=self.config.get('dataformat_ohlcv', 'json'),
)
# Download informative pairs too
res = defaultdict(list)
for p, t in self.strategy.informative_pairs():
res[t].append(p)
for timeframe, inf_pairs in res.items():
timerange_startup = deepcopy(self._timerange)
timerange_startup.subtract_start(timeframe_to_seconds(
timeframe) * self.strategy.startup_candle_count)
refresh_data(
datadir=self.config['datadir'],
pairs=inf_pairs,
exchange=self.exchange,
timeframe=timeframe,
timerange=timerange_startup,
data_format=self.config.get('dataformat_ohlcv', 'json'),
)
data = load_data(
datadir=self.config['datadir'],
@ -121,8 +148,11 @@ class Edge:
self._cached_pairs = {}
logger.critical("No data found. Edge is stopped ...")
return False
# Fake run-mode to Edge
prior_rm = self.config['runmode']
self.config['runmode'] = RunMode.EDGE
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
self.config['runmode'] = prior_rm
# Print timeframe
min_date, max_date = get_timerange(preprocessed)
@ -179,7 +209,7 @@ class Edge:
if pair in self._cached_pairs:
return self._cached_pairs[pair].stoploss
else:
logger.warning('tried to access stoploss of a non-existing pair, '
logger.warning(f'Tried to access stoploss of non-existing pair {pair}, '
'strategy stoploss is returned instead.')
return self.strategy.stoploss
@ -210,7 +240,7 @@ class Edge:
return self._final_pairs
def accepted_pairs(self) -> list:
def accepted_pairs(self) -> List[Dict[str, Any]]:
"""
return a list of accepted pairs along with their winrate, expectancy and stoploss
"""

6
freqtrade/exchange/__init__.py
View File

@ -8,10 +8,12 @@ from freqtrade.exchange.binance import Binance
from freqtrade.exchange.bittrex import Bittrex
from freqtrade.exchange.bybit import Bybit
from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
get_exchange_bad_reason, is_exchange_bad,
is_exchange_known_ccxt, is_exchange_officially_supported,
market_is_active, timeframe_to_minutes, timeframe_to_msecs,
timeframe_to_next_date, timeframe_to_prev_date,
timeframe_to_seconds)
timeframe_to_seconds, validate_exchange,
validate_exchanges)
from freqtrade.exchange.ftx import Ftx
from freqtrade.exchange.hitbtc import Hitbtc
from freqtrade.exchange.kraken import Kraken
from freqtrade.exchange.kucoin import Kucoin

2
freqtrade/exchange/binance.py
View File

@ -52,7 +52,7 @@ class Binance(Exchange):
'In stoploss limit order, stop price should be more than limit price')
if self._config['dry_run']:
dry_order = self.dry_run_order(
dry_order = self.create_dry_run_order(
pair, ordertype, "sell", amount, stop_price)
return dry_order

4
freqtrade/exchange/bittrex.py
View File

@ -12,10 +12,6 @@ class Bittrex(Exchange):
"""
Bittrex exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
Please note that this exchange is not included in the list of exchanges
officially supported by the Freqtrade development team. So some features
may still not work as expected.
"""
_ft_has: Dict = {

95
freqtrade/exchange/common.py
View File

@ -18,78 +18,8 @@ BAD_EXCHANGES = {
"bitmex": "Various reasons.",
"bitstamp": "Does not provide history. "
"Details in https://github.com/freqtrade/freqtrade/issues/1983",
"hitbtc": "This API cannot be used with Freqtrade. "
"Use `hitbtc2` exchange id to access this exchange.",
"phemex": "Does not provide history. ",
"poloniex": "Does not provide fetch_order endpoint to fetch both open and closed orders.",
**dict.fromkeys([
'adara',
'anxpro',
'bigone',
'coinbase',
'coinexchange',
'coinmarketcap',
'lykke',
'xbtce',
], "Does not provide timeframes. ccxt fetchOHLCV: False"),
**dict.fromkeys([
'bcex',
'bit2c',
'bitbay',
'bitflyer',
'bitforex',
'bithumb',
'bitso',
'bitstamp1',
'bl3p',
'braziliex',
'btcbox',
'btcchina',
'btctradeim',
'btctradeua',
'bxinth',
'chilebit',
'coincheck',
'coinegg',
'coinfalcon',
'coinfloor',
'coingi',
'coinmate',
'coinone',
'coinspot',
'coolcoin',
'crypton',
'deribit',
'exmo',
'exx',
'flowbtc',
'foxbit',
'fybse',
# 'hitbtc',
'ice3x',
'independentreserve',
'indodax',
'itbit',
'lakebtc',
'latoken',
'liquid',
'livecoin',
'luno',
'mixcoins',
'negociecoins',
'nova',
'paymium',
'southxchange',
'stronghold',
'surbitcoin',
'therock',
'tidex',
'vaultoro',
'vbtc',
'virwox',
'yobit',
'zaif',
], "Does not provide timeframes. ccxt fetchOHLCV: emulated"),
}
MAP_EXCHANGE_CHILDCLASS = {
@ -98,6 +28,29 @@ MAP_EXCHANGE_CHILDCLASS = {
}
EXCHANGE_HAS_REQUIRED = [
# Required / private
'fetchOrder',
'cancelOrder',
'createOrder',
# 'createLimitOrder', 'createMarketOrder',
'fetchBalance',
# Public endpoints
'loadMarkets',
'fetchOHLCV',
]
EXCHANGE_HAS_OPTIONAL = [
# Private
'fetchMyTrades', # Trades for order - fee detection
# Public
'fetchOrderBook', 'fetchL2OrderBook', 'fetchTicker', # OR for pricing
'fetchTickers', # For volumepairlist?
'fetchTrades', # Downloading trades data
]
def calculate_backoff(retrycount, max_retries):
"""
Calculate backoff
@ -140,7 +93,7 @@ def retrier(_func=None, retries=API_RETRY_COUNT):
logger.warning('retrying %s() still for %s times', f.__name__, count)
count -= 1
kwargs.update({'count': count})
if isinstance(ex, DDosProtection) or isinstance(ex, RetryableOrderError):
if isinstance(ex, (DDosProtection, RetryableOrderError)):
# increasing backoff
backoff_delay = calculate_backoff(count + 1, retries)
logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}")

142
freqtrade/exchange/exchange.py
View File

@ -14,6 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
import arrow
import ccxt
import ccxt.async_support as ccxt_async
from cachetools import TTLCache
from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE,
decimal_to_precision)
from pandas import DataFrame
@ -23,7 +24,8 @@ from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
InvalidOrderException, OperationalException, RetryableOrderError,
TemporaryError)
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier,
from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier,
retrier_async)
from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@ -57,11 +59,13 @@ class Exchange:
_ft_has_default: Dict = {
"stoploss_on_exchange": False,
"order_time_in_force": ["gtc"],
"ohlcv_params": {},
"ohlcv_candle_limit": 500,
"ohlcv_partial_candle": True,
"trades_pagination": "time", # Possible are "time" or "id"
"trades_pagination_arg": "since",
"l2_limit_range": None,
"l2_limit_range_required": True, # Allow Empty L2 limit (kucoin)
}
_ft_has: Dict = {}
@ -82,6 +86,9 @@ class Exchange:
# Timestamp of last markets refresh
self._last_markets_refresh: int = 0
# Cache for 10 minutes ...
self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10)
# Holds candles
self._klines: Dict[Tuple[str, str], DataFrame] = {}
@ -357,7 +364,6 @@ class Exchange:
invalid_pairs = []
for pair in extended_pairs:
# Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
# TODO: add a support for having coins in BTC/USDT format
if self.markets and pair not in self.markets:
raise OperationalException(
f'Pair {pair} is not available on {self.name}. '
@ -460,7 +466,7 @@ class Exchange:
def amount_to_precision(self, pair: str, amount: float) -> float:
'''
Returns the amount to buy or sell to a precision the Exchange accepts
Reimplementation of ccxt internal methods - ensuring we can test the result is correct
Re-implementation of ccxt internal methods - ensuring we can test the result is correct
based on our definitions.
'''
if self.markets[pair]['precision']['amount']:
@ -474,7 +480,7 @@ class Exchange:
def price_to_precision(self, pair: str, price: float) -> float:
'''
Returns the price rounded up to the precision the Exchange accepts.
Partial Reimplementation of ccxt internal method decimal_to_precision(),
Partial Re-implementation of ccxt internal method decimal_to_precision(),
which does not support rounding up
TODO: If ccxt supports ROUND_UP for decimal_to_precision(), we could remove this and
align with amount_to_precision().
@ -533,7 +539,9 @@ class Exchange:
# reserve some percent defined in config (5% default) + stoploss
amount_reserve_percent = 1.0 + self._config.get('amount_reserve_percent',
DEFAULT_AMOUNT_RESERVE_PERCENT)
amount_reserve_percent += abs(stoploss)
amount_reserve_percent = (
amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5
)
# it should not be more than 50%
amount_reserve_percent = max(min(amount_reserve_percent, 1.5), 1)
@ -542,8 +550,8 @@ class Exchange:
# See also #2575 at github.
return max(min_stake_amounts) * amount_reserve_percent
def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
rate: float, params: Dict = {}) -> Dict[str, Any]:
def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
rate: float, params: Dict = {}) -> Dict[str, Any]:
order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
_amount = self.amount_to_precision(pair, amount)
dry_order = {
@ -617,7 +625,7 @@ class Exchange:
rate: float, time_in_force: str) -> Dict:
if self._config['dry_run']:
dry_order = self.dry_run_order(pair, ordertype, "buy", amount, rate)
dry_order = self.create_dry_run_order(pair, ordertype, "buy", amount, rate)
return dry_order
params = self._params.copy()
@ -630,7 +638,7 @@ class Exchange:
rate: float, time_in_force: str = 'gtc') -> Dict:
if self._config['dry_run']:
dry_order = self.dry_run_order(pair, ordertype, "sell", amount, rate)
dry_order = self.create_dry_run_order(pair, ordertype, "sell", amount, rate)
return dry_order
params = self._params.copy()
@ -659,23 +667,8 @@ class Exchange:
raise OperationalException(f"stoploss is not implemented for {self.name}.")
@retrier
def get_balance(self, currency: str) -> float:
if self._config['dry_run']:
return self._config['dry_run_wallet']
# ccxt exception is already handled by get_balances
balances = self.get_balances()
balance = balances.get(currency)
if balance is None:
raise TemporaryError(
f'Could not get {currency} balance due to malformed exchange response: {balances}')
return balance['free']
@retrier
def get_balances(self) -> dict:
if self._config['dry_run']:
return {}
try:
balances = self._api.fetch_balance()
@ -695,9 +688,19 @@ class Exchange:
raise OperationalException(e) from e
@retrier
def get_tickers(self) -> Dict:
def get_tickers(self, cached: bool = False) -> Dict:
"""
:param cached: Allow cached result
:return: fetch_tickers result
"""
if cached:
tickers = self._fetch_tickers_cache.get('fetch_tickers')
if tickers:
return tickers
try:
return self._api.fetch_tickers()
tickers = self._api.fetch_tickers()
self._fetch_tickers_cache['fetch_tickers'] = tickers
return tickers
except ccxt.NotSupported as e:
raise OperationalException(
f'Exchange {self._api.name} does not support fetching tickers in batch. '
@ -806,7 +809,7 @@ class Exchange:
# Gather coroutines to run
for pair, timeframe in set(pair_list):
if (not ((pair, timeframe) in self._klines)
if (((pair, timeframe) not in self._klines)
or self._now_is_time_to_refresh(pair, timeframe)):
input_coroutines.append(self._async_get_candle_history(pair, timeframe,
since_ms=since_ms))
@ -860,10 +863,11 @@ class Exchange:
"Fetching pair %s, interval %s, since %s %s...",
pair, timeframe, since_ms, s
)
params = self._ft_has.get('ohlcv_params', {})
data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe,
since=since_ms,
limit=self.ohlcv_candle_limit(timeframe))
limit=self.ohlcv_candle_limit(timeframe),
params=params)
# Some exchanges sort OHLCV in ASC order and others in DESC.
# Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last)
@ -958,7 +962,7 @@ class Exchange:
while True:
t = await self._async_fetch_trades(pair,
params={self._trades_pagination_arg: from_id})
if len(t):
if t:
# Skip last id since its the key for the next call
trades.extend(t[:-1])
if from_id == t[-1][1] or t[-1][0] > until:
@ -990,7 +994,7 @@ class Exchange:
# DEFAULT_TRADES_COLUMNS: 1 -> id
while True:
t = await self._async_fetch_trades(pair, since=since)
if len(t):
if t:
since = t[-1][0]
trades.extend(t)
# Reached the end of the defined-download period
@ -1116,6 +1120,27 @@ class Exchange:
return order
def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict:
"""
Cancel stoploss order returning a result.
Creates a fake result if cancel order returns a non-usable result
and fetch_order does not work (certain exchanges don't return cancelled orders)
:param order_id: stoploss-order-id to cancel
:param pair: Pair corresponding to order_id
:param amount: Amount to use for fake response
:return: Result from either cancel_order if usable, or fetch_order
"""
corder = self.cancel_stoploss_order(order_id, pair)
if self.is_cancel_order_result_suitable(corder):
return corder
try:
order = self.fetch_stoploss_order(order_id, pair)
except InvalidOrderException:
logger.warning(f"Could not fetch cancelled stoploss order {order_id}.")
order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}}
return order
@retrier(retries=API_FETCH_ORDER_RETRY_COUNT)
def fetch_order(self, order_id: str, pair: str) -> Dict:
if self._config['dry_run']:
@ -1157,14 +1182,20 @@ class Exchange:
return self.fetch_order(order_id, pair)
@staticmethod
def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]]):
def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]],
range_required: bool = True):
"""
Get next greater value in the list.
Used by fetch_l2_order_book if the api only supports a limited range
"""
if not limit_range:
return limit
return min([x for x in limit_range if limit <= x] + [max(limit_range)])
result = min([x for x in limit_range if limit <= x] + [max(limit_range)])
if not range_required and limit > result:
# Range is not required - we can use None as parameter.
return None
return result
@retrier
def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
@ -1174,7 +1205,8 @@ class Exchange:
Returns a dict in the format
{'asks': [price, volume], 'bids': [price, volume]}
"""
limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'])
limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'],
self._ft_has['l2_limit_range_required'])
try:
return self._api.fetch_l2_order_book(pair, limit1)
@ -1228,6 +1260,9 @@ class Exchange:
except ccxt.BaseError as e:
raise OperationalException(e) from e
def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
return order['id']
@retrier
def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1,
price: float = 1, taker_or_maker: str = 'maker') -> float:
@ -1306,14 +1341,6 @@ class Exchange:
self.calculate_fee_rate(order))
def is_exchange_bad(exchange_name: str) -> bool:
return exchange_name in BAD_EXCHANGES
def get_exchange_bad_reason(exchange_name: str) -> str:
return BAD_EXCHANGES.get(exchange_name, "")
def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool:
return exchange_name in ccxt_exchanges(ccxt_module)
@ -1334,7 +1361,36 @@ def available_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:
Return exchanges available to the bot, i.e. non-bad exchanges in the ccxt list
"""
exchanges = ccxt_exchanges(ccxt_module)
return [x for x in exchanges if not is_exchange_bad(x)]
return [x for x in exchanges if validate_exchange(x)[0]]
def validate_exchange(exchange: str) -> Tuple[bool, str]:
ex_mod = getattr(ccxt, exchange.lower())()
if not ex_mod or not ex_mod.has:
return False, ''
missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True]
if missing:
return False, f"missing: {', '.join(missing)}"
missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)]
if exchange.lower() in BAD_EXCHANGES:
return False, BAD_EXCHANGES.get(exchange.lower(), '')
if missing_opt:
return True, f"missing opt: {', '.join(missing_opt)}"
return True, ''
def validate_exchanges(all_exchanges: bool) -> List[Tuple[str, bool, str]]:
"""
:return: List of tuples with exchangename, valid, reason.
"""
exchanges = ccxt_exchanges() if all_exchanges else available_exchanges()
exchanges_valid = [
(e, *validate_exchange(e)) for e in exchanges
]
return exchanges_valid
def timeframe_to_seconds(timeframe: str) -> int:

11
freqtrade/exchange/ftx.py
View File

@ -8,6 +8,7 @@ from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, Invali
OperationalException, TemporaryError)
from freqtrade.exchange import Exchange
from freqtrade.exchange.common import API_FETCH_ORDER_RETRY_COUNT, retrier
from freqtrade.misc import safe_value_fallback2
logger = logging.getLogger(__name__)
@ -53,7 +54,7 @@ class Ftx(Exchange):
stop_price = self.price_to_precision(pair, stop_price)
if self._config['dry_run']:
dry_order = self.dry_run_order(
dry_order = self.create_dry_run_order(
pair, ordertype, "sell", amount, stop_price)
return dry_order
@ -63,10 +64,11 @@ class Ftx(Exchange):
# set orderPrice to place limit order, otherwise it's a market order
params['orderPrice'] = limit_rate
params['stopPrice'] = stop_price
amount = self.amount_to_precision(pair, amount)
order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
amount=amount, price=stop_price, params=params)
amount=amount, params=params)
logger.info('stoploss order added for %s. '
'stop price: %s.', pair, stop_price)
return order
@ -134,3 +136,8 @@ class Ftx(Exchange):
f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e
except ccxt.BaseError as e:
raise OperationalException(e) from e
def get_order_id_conditional(self, order: Dict[str, Any]) -> str:
if order['type'] == 'stop':
return safe_value_fallback2(order['info'], order, 'orderId', 'id')
return order['id']

24
freqtrade/exchange/hitbtc.py Normal file
View File

@ -0,0 +1,24 @@
import logging
from typing import Dict
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)
class Hitbtc(Exchange):
"""
Hitbtc exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
Please note that this exchange is not included in the list of exchanges
officially supported by the Freqtrade development team. So some features
may still not work as expected.
"""
# fetchCurrencies API point requires authentication for Hitbtc,
_ft_has: Dict = {
"ohlcv_candle_limit": 1000,
"ohlcv_params": {"sort": "DESC"}
}

4
freqtrade/exchange/kraken.py
View File

@ -53,6 +53,8 @@ class Kraken(Exchange):
# x["side"], x["amount"],
) for x in orders]
for bal in balances:
if not isinstance(balances[bal], dict):
continue
balances[bal]['used'] = sum(order[1] for order in order_list if order[0] == bal)
balances[bal]['free'] = balances[bal]['total'] - balances[bal]['used']
@ -92,7 +94,7 @@ class Kraken(Exchange):
stop_price = self.price_to_precision(pair, stop_price)
if self._config['dry_run']:
dry_order = self.dry_run_order(
dry_order = self.create_dry_run_order(
pair, ordertype, "sell", amount, stop_price)
return dry_order

24
freqtrade/exchange/kucoin.py Normal file
View File

@ -0,0 +1,24 @@
""" Kucoin exchange subclass """
import logging
from typing import Dict
from freqtrade.exchange import Exchange
logger = logging.getLogger(__name__)
class Kucoin(Exchange):
"""
Kucoin exchange class. Contains adjustments needed for Freqtrade to work
with this exchange.
Please note that this exchange is not included in the list of exchanges
officially supported by the Freqtrade development team. So some features
may still not work as expected.
"""
_ft_has: Dict = {
"l2_limit_range": [20, 100],
"l2_limit_range_required": False,
}

141
freqtrade/freqtradebot.py
View File

@ -28,7 +28,7 @@ from freqtrade.plugins.protectionmanager import ProtectionManager
from freqtrade.resolvers import ExchangeResolver, StrategyResolver
from freqtrade.rpc import RPCManager, RPCMessageType
from freqtrade.state import State
from freqtrade.strategy.interface import IStrategy, SellType
from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType
from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
from freqtrade.wallets import Wallets
@ -113,7 +113,7 @@ class FreqtradeBot(LoggingMixin):
via RPC about changes in the bot status.
"""
self.rpc.send_msg({
'type': RPCMessageType.STATUS_NOTIFICATION,
'type': RPCMessageType.STATUS,
'status': msg
})
@ -187,7 +187,7 @@ class FreqtradeBot(LoggingMixin):
if self.get_free_open_trades():
self.enter_positions()
Trade.session.flush()
Trade.query.session.flush()
def process_stopped(self) -> None:
"""
@ -205,7 +205,7 @@ class FreqtradeBot(LoggingMixin):
if len(open_trades) != 0:
msg = {
'type': RPCMessageType.WARNING_NOTIFICATION,
'type': RPCMessageType.WARNING,
'status': f"{len(open_trades)} open trades active.\n\n"
f"Handle these trades manually on {self.exchange.name}, "
f"or '/start' the bot again and use '/stopbuy' "
@ -225,7 +225,7 @@ class FreqtradeBot(LoggingMixin):
# Calculating Edge positioning
if self.edge:
self.edge.calculate()
self.edge.calculate(_whitelist)
_whitelist = self.edge.adjust(_whitelist)
if trades:
@ -267,7 +267,7 @@ class FreqtradeBot(LoggingMixin):
def update_closed_trades_without_assigned_fees(self):
"""
Update closed trades without close fees assigned.
Only acts when Orders are in the database, otherwise the last orderid is unknown.
Only acts when Orders are in the database, otherwise the last order-id is unknown.
"""
if self.config['dry_run']:
# Updating open orders in dry-run does not make sense and will fail.
@ -378,7 +378,7 @@ class FreqtradeBot(LoggingMixin):
if lock:
self.log_once(f"Global pairlock active until "
f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}. "
"Not creating new trades.", logger.info)
f"Not creating new trades, reason: {lock.reason}.", logger.info)
else:
self.log_once("Global pairlock active. Not creating new trades.", logger.info)
return trades_created
@ -410,9 +410,7 @@ class FreqtradeBot(LoggingMixin):
bid_strategy = self.config.get('bid_strategy', {})
if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False):
logger.info(
f"Getting price from order book {bid_strategy['price_side'].capitalize()} side."
)
order_book_top = bid_strategy.get('order_book_top', 1)
order_book = self.exchange.fetch_l2_order_book(pair, order_book_top)
logger.debug('order_book %s', order_book)
@ -425,7 +423,8 @@ class FreqtradeBot(LoggingMixin):
f"Orderbook: {order_book}"
)
raise PricingError from e
logger.info(f'...top {order_book_top} order book buy rate {rate_from_l2:.8f}')
logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side "
f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}")
used_rate = rate_from_l2
else:
logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price")
@ -457,7 +456,8 @@ class FreqtradeBot(LoggingMixin):
lock = PairLocks.get_pair_longest_lock(pair, nowtime)
if lock:
self.log_once(f"Pair {pair} is still locked until "
f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}.",
f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} "
f"due to {lock.reason}.",
logger.info)
else:
self.log_once(f"Pair {pair} is still locked.", logger.info)
@ -473,25 +473,22 @@ class FreqtradeBot(LoggingMixin):
(buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
if buy and not sell:
stake_amount = self.wallets.get_trade_stake_amount(pair, self.get_free_open_trades(),
self.edge)
stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
if not stake_amount:
logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
return False
logger.info(f"Buy signal found: about create a new trade with stake_amount: "
logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: "
f"{stake_amount} ...")
bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {})
if ((bid_check_dom.get('enabled', False)) and
(bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
if self._check_depth_of_market_buy(pair, bid_check_dom):
logger.info(f'Executing Buy for {pair}.')
return self.execute_buy(pair, stake_amount)
else:
return False
logger.info(f'Executing Buy for {pair}')
return self.execute_buy(pair, stake_amount)
else:
return False
@ -555,7 +552,7 @@ class FreqtradeBot(LoggingMixin):
if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested,
time_in_force=time_in_force):
time_in_force=time_in_force, current_time=datetime.now(timezone.utc)):
logger.info(f"User requested abortion of buying {pair}")
return False
amount = self.exchange.amount_to_precision(pair, amount)
@ -621,8 +618,8 @@ class FreqtradeBot(LoggingMixin):
if order_status == 'closed':
self.update_trade_state(trade, order_id, order)
Trade.session.add(trade)
Trade.session.flush()
Trade.query.session.add(trade)
Trade.query.session.flush()
# Updating wallets
self.wallets.update()
@ -633,11 +630,11 @@ class FreqtradeBot(LoggingMixin):
def _notify_buy(self, trade: Trade, order_type: str) -> None:
"""
Sends rpc notification when a buy occured.
Sends rpc notification when a buy occurred.
"""
msg = {
'trade_id': trade.id,
'type': RPCMessageType.BUY_NOTIFICATION,
'type': RPCMessageType.BUY,
'exchange': self.exchange.name.capitalize(),
'pair': trade.pair,
'limit': trade.open_rate,
@ -655,13 +652,13 @@ class FreqtradeBot(LoggingMixin):
def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
"""
Sends rpc notification when a buy cancel occured.
Sends rpc notification when a buy cancel occurred.
"""
current_rate = self.get_buy_rate(trade.pair, False)
msg = {
'trade_id': trade.id,
'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
'type': RPCMessageType.BUY_CANCEL,
'exchange': self.exchange.name.capitalize(),
'pair': trade.pair,
'limit': trade.open_rate,
@ -678,6 +675,21 @@ class FreqtradeBot(LoggingMixin):
# Send the message
self.rpc.send_msg(msg)
def _notify_buy_fill(self, trade: Trade) -> None:
msg = {
'trade_id': trade.id,
'type': RPCMessageType.BUY_FILL,
'exchange': self.exchange.name.capitalize(),
'pair': trade.pair,
'open_rate': trade.open_rate,
'stake_amount': trade.stake_amount,
'stake_currency': self.config['stake_currency'],
'fiat_currency': self.config.get('fiat_display_currency', None),
'amount': trade.amount,
'open_date': trade.open_date,
}
self.rpc.send_msg(msg)
#
# SELL / exit positions / close trades logic and methods
#
@ -701,7 +713,7 @@ class FreqtradeBot(LoggingMixin):
except DependencyException as exception:
logger.warning('Unable to sell trade %s: %s', trade.pair, exception)
# Updating wallets if any trade occured
# Updating wallets if any trade occurred
if trades_closed:
self.wallets.update()
@ -838,7 +850,8 @@ class FreqtradeBot(LoggingMixin):
trade.stoploss_order_id = None
logger.error(f'Unable to place a stoploss order on exchange. {e}')
logger.warning('Selling the trade forcefully')
self.execute_sell(trade, trade.stop_loss, sell_reason=SellType.EMERGENCY_SELL)
self.execute_sell(trade, trade.stop_loss, sell_reason=SellCheckTuple(
sell_type=SellType.EMERGENCY_SELL))
except ExchangeError:
trade.stoploss_order_id = None
@ -919,14 +932,15 @@ class FreqtradeBot(LoggingMixin):
:return: None
"""
if self.exchange.stoploss_adjust(trade.stop_loss, order):
# we check if the update is neccesary
# we check if the update is necessary
update_beat = self.strategy.order_types.get('stoploss_on_exchange_interval', 60)
if (datetime.utcnow() - trade.stoploss_last_update).total_seconds() >= update_beat:
# cancelling the current stoploss on exchange first
logger.info(f"Cancelling current stoploss on exchange for pair {trade.pair} "
f"(orderid:{order['id']}) in order to add another one ...")
try:
co = self.exchange.cancel_stoploss_order(order['id'], trade.pair)
co = self.exchange.cancel_stoploss_order_with_result(order['id'], trade.pair,
trade.amount)
trade.update_order(co)
except InvalidOrderException:
logger.exception(f"Could not cancel stoploss order {order['id']} "
@ -949,7 +963,7 @@ class FreqtradeBot(LoggingMixin):
if should_sell.sell_flag:
logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}')
self.execute_sell(trade, sell_rate, should_sell.sell_type)
self.execute_sell(trade, sell_rate, should_sell)
return True
return False
@ -960,15 +974,16 @@ class FreqtradeBot(LoggingMixin):
timeout = self.config.get('unfilledtimeout', {}).get(side)
ordertime = arrow.get(order['datetime']).datetime
if timeout is not None:
timeout_threshold = arrow.utcnow().shift(minutes=-timeout).datetime
timeout_unit = self.config.get('unfilledtimeout', {}).get('unit', 'minutes')
timeout_kwargs = {timeout_unit: -timeout}
timeout_threshold = arrow.utcnow().shift(**timeout_kwargs).datetime
return (order['status'] == 'open' and order['side'] == side
and ordertime < timeout_threshold)
return False
def check_handle_timedout(self) -> None:
"""
Check if any orders are timed out and cancel if neccessary
Check if any orders are timed out and cancel if necessary
:param timeoutvalue: Number of minutes until order is considered timed out
:return: None
"""
@ -1030,6 +1045,16 @@ class FreqtradeBot(LoggingMixin):
# Cancelled orders may have the status of 'canceled' or 'closed'
if order['status'] not in ('cancelled', 'canceled', 'closed'):
filled_val = order.get('filled', 0.0) or 0.0
filled_stake = filled_val * trade.open_rate
minstake = self.exchange.get_min_pair_stake_amount(
trade.pair, trade.open_rate, self.strategy.stoploss)
if filled_val > 0 and filled_stake < minstake:
logger.warning(
f"Order {trade.open_order_id} for {trade.pair} not cancelled, "
f"as the filled amount of {filled_val} would result in an unsellable trade.")
return False
corder = self.exchange.cancel_order_with_result(trade.open_order_id, trade.pair,
trade.amount)
# Avoid race condition where the order could not be cancelled coz its already filled.
@ -1138,16 +1163,16 @@ class FreqtradeBot(LoggingMixin):
raise DependencyException(
f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}")
def execute_sell(self, trade: Trade, limit: float, sell_reason: SellType) -> bool:
def execute_sell(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool:
"""
Executes a limit sell for the given trade and limit
:param trade: Trade instance
:param limit: limit rate for the sell order
:param sellreason: Reason the sell was triggered
:param sell_reason: Reason the sell was triggered
:return: True if it succeeds (supported) False (not supported)
"""
sell_type = 'sell'
if sell_reason in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
if sell_reason.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
sell_type = 'stoploss'
# if stoploss is on exchange and we are on dry_run mode,
@ -1159,15 +1184,17 @@ class FreqtradeBot(LoggingMixin):
# First cancelling stoploss on exchange ...
if self.strategy.order_types.get('stoploss_on_exchange') and trade.stoploss_order_id:
try:
self.exchange.cancel_stoploss_order(trade.stoploss_order_id, trade.pair)
co = self.exchange.cancel_stoploss_order_with_result(trade.stoploss_order_id,
trade.pair, trade.amount)
trade.update_order(co)
except InvalidOrderException:
logger.exception(f"Could not cancel stoploss order {trade.stoploss_order_id}")
order_type = self.strategy.order_types[sell_type]
if sell_reason == SellType.EMERGENCY_SELL:
if sell_reason.sell_type == SellType.EMERGENCY_SELL:
# Emergency sells (default to market!)
order_type = self.strategy.order_types.get("emergencysell", "market")
if sell_reason == SellType.FORCE_SELL:
if sell_reason.sell_type == SellType.FORCE_SELL:
# Force sells (default to the sell_type defined in the strategy,
# but we allow this value to be changed)
order_type = self.strategy.order_types.get("forcesell", order_type)
@ -1177,8 +1204,8 @@ class FreqtradeBot(LoggingMixin):
if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)(
pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit,
time_in_force=time_in_force,
sell_reason=sell_reason.value):
time_in_force=time_in_force, sell_reason=sell_reason.sell_reason,
current_time=datetime.now(timezone.utc)):
logger.info(f"User requested abortion of selling {trade.pair}")
return False
@ -1201,13 +1228,13 @@ class FreqtradeBot(LoggingMixin):
trade.open_order_id = order['id']
trade.sell_order_status = ''
trade.close_rate_requested = limit
trade.sell_reason = sell_reason.value
trade.sell_reason = sell_reason.sell_reason
# In case of market sell orders the order can be closed immediately
if order.get('status', 'unknown') == 'closed':
self.update_trade_state(trade, trade.open_order_id, order)
Trade.session.flush()
Trade.query.session.flush()
# Lock pair for one candle to prevent immediate rebuys
# Lock pair for one candle to prevent immediate re-buys
self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
reason='Auto lock')
@ -1215,19 +1242,20 @@ class FreqtradeBot(LoggingMixin):
return True
def _notify_sell(self, trade: Trade, order_type: str) -> None:
def _notify_sell(self, trade: Trade, order_type: str, fill: bool = False) -> None:
"""
Sends rpc notification when a sell occured.
Sends rpc notification when a sell occurred.
"""
profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
profit_trade = trade.calc_profit(rate=profit_rate)
# Use cached rates here - it was updated seconds ago.
current_rate = self.get_sell_rate(trade.pair, False)
current_rate = self.get_sell_rate(trade.pair, False) if not fill else None
profit_ratio = trade.calc_profit_ratio(profit_rate)
gain = "profit" if profit_ratio > 0 else "loss"
msg = {
'type': RPCMessageType.SELL_NOTIFICATION,
'type': (RPCMessageType.SELL_FILL if fill
else RPCMessageType.SELL),
'trade_id': trade.id,
'exchange': trade.exchange.capitalize(),
'pair': trade.pair,
@ -1236,6 +1264,7 @@ class FreqtradeBot(LoggingMixin):
'order_type': order_type,
'amount': trade.amount,
'open_rate': trade.open_rate,
'close_rate': trade.close_rate,
'current_rate': current_rate,
'profit_amount': profit_trade,
'profit_ratio': profit_ratio,
@ -1256,7 +1285,7 @@ class FreqtradeBot(LoggingMixin):
def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None:
"""
Sends rpc notification when a sell cancel occured.
Sends rpc notification when a sell cancel occurred.
"""
if trade.sell_order_status == reason:
return
@ -1270,7 +1299,7 @@ class FreqtradeBot(LoggingMixin):
gain = "profit" if profit_ratio > 0 else "loss"
msg = {
'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
'type': RPCMessageType.SELL_CANCEL,
'trade_id': trade.id,
'exchange': trade.exchange.capitalize(),
'pair': trade.pair,
@ -1309,7 +1338,7 @@ class FreqtradeBot(LoggingMixin):
Handles closing both buy and sell orders.
:param trade: Trade object of the trade we're analyzing
:param order_id: Order-id of the order we're analyzing
:param action_order: Already aquired order object
:param action_order: Already acquired order object
:return: True if order has been cancelled without being filled partially, False otherwise
"""
if not order_id:
@ -1347,9 +1376,15 @@ class FreqtradeBot(LoggingMixin):
# Updating wallets when order is closed
if not trade.is_open:
if not stoploss_order and not trade.open_order_id:
self._notify_sell(trade, '', True)
self.protections.stop_per_pair(trade.pair)
self.protections.global_stop()
self.wallets.update()
elif not trade.open_order_id:
# Buy fill
self._notify_buy_fill(trade)
return False
def apply_fee_conditional(self, trade: Trade, trade_base_currency: str,
@ -1373,7 +1408,7 @@ class FreqtradeBot(LoggingMixin):
def get_real_amount(self, trade: Trade, order: Dict) -> float:
"""
Detect and update trade fee.
Calls trade.update_fee() uppon correct detection.
Calls trade.update_fee() upon correct detection.
Returns modified amount if the fee was taken from the destination currency.
Necessary for exchanges which charge fees in base currency (e.g. binance)
:return: identical (or new) amount for the trade
@ -1406,8 +1441,8 @@ class FreqtradeBot(LoggingMixin):
"""
fee-detection fallback to Trades. Parses result of fetch_my_trades to get correct fee.
"""
trades = self.exchange.get_trades_for_order(order['id'], trade.pair,
trade.open_date)
trades = self.exchange.get_trades_for_order(self.exchange.get_order_id_conditional(order),
trade.pair, trade.open_date)
if len(trades) == 0:
logger.info("Applying fee on amount for %s failed: myTrade-Dict empty found", trade)

15
freqtrade/misc.py
View File

@ -6,7 +6,7 @@ import logging
import re
from datetime import datetime
from pathlib import Path
from typing import Any
from typing import Any, Iterator, List
from typing.io import IO
import rapidjson
@ -81,7 +81,7 @@ def json_load(datafile: IO) -> Any:
"""
load data with rapidjson
Use this to have a consistent experience,
sete number_mode to "NM_NATIVE" for greatest speed
set number_mode to "NM_NATIVE" for greatest speed
"""
return rapidjson.load(datafile, number_mode=rapidjson.NM_NATIVE)
@ -202,3 +202,14 @@ def render_template_with_fallback(templatefile: str, templatefallbackfile: str,
return render_template(templatefile, arguments)
except TemplateNotFound:
return render_template(templatefallbackfile, arguments)
def chunks(lst: List[Any], n: int) -> Iterator[List[Any]]:
"""
Split lst into chunks of the size n.
:param lst: list to split into chunks
:param n: number of max elements per chunk
:return: None
"""
for chunk in range(0, len(lst), n):
yield (lst[chunk:chunk + n])

135
freqtrade/optimize/backtesting.py
View File

@ -15,7 +15,7 @@ from freqtrade.configuration import TimeRange, remove_credentials, validate_conf
from freqtrade.constants import DATETIME_PRINT_FORMAT
from freqtrade.data import history
from freqtrade.data.btanalysis import trade_list_to_dataframe
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.dataprovider import DataProvider
from freqtrade.exceptions import DependencyException, OperationalException
from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
@ -63,9 +63,7 @@ class Backtesting:
self.all_results: Dict[str, Dict] = {}
self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config)
dataprovider = DataProvider(self.config, self.exchange)
IStrategy.dp = dataprovider
self.dataprovider = DataProvider(self.config, None)
if self.config.get('strategy_list', None):
for strat in list(self.config['strategy_list']):
@ -96,7 +94,7 @@ class Backtesting:
"PrecisionFilter not allowed for backtesting multiple strategies."
)
dataprovider.add_pairlisthandler(self.pairlists)
self.dataprovider.add_pairlisthandler(self.pairlists)
self.pairlists.refresh_pairlist()
if len(self.pairlists.whitelist) == 0:
@ -112,15 +110,11 @@ class Backtesting:
PairLocks.timeframe = self.config['timeframe']
PairLocks.use_db = False
PairLocks.reset_locks()
if self.config.get('enable_protections', False):
self.protections = ProtectionManager(self.config)
self.wallets = Wallets(self.config, self.exchange, log=False)
# Get maximum required startup period
self.required_startup = max([strat.startup_candle_count for strat in self.strategylist])
# Load one (first) strategy
self._set_strategy(self.strategylist[0])
def __del__(self):
LoggingMixin.show_output = True
@ -132,10 +126,17 @@ class Backtesting:
Load strategy into backtesting
"""
self.strategy: IStrategy = strategy
strategy.dp = self.dataprovider
# Set stoploss_on_exchange to false for backtesting,
# since a "perfect" stoploss-sell is assumed anyway
# And the regular "stoploss" function would not apply to that case
self.strategy.order_types['stoploss_on_exchange'] = False
if self.config.get('enable_protections', False):
conf = self.config
if hasattr(strategy, 'protections'):
conf = deepcopy(conf)
conf['protections'] = strategy.protections
self.protections = ProtectionManager(conf)
def load_bt_data(self) -> Tuple[Dict[str, DataFrame], TimeRange]:
"""
@ -159,7 +160,7 @@ class Backtesting:
logger.info(f'Loading data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
f'({(max_date - min_date).days} days).')
# Adjust startts forward if not enough data is available
timerange.adjust_start_if_necessary(timeframe_to_seconds(self.timeframe),
@ -176,6 +177,8 @@ class Backtesting:
Trade.use_db = False
PairLocks.reset_locks()
Trade.reset_trades()
self.rejected_trades = 0
self.dataprovider.clear_cache()
def _get_ohlcv_as_lists(self, processed: Dict[str, DataFrame]) -> Dict[str, Tuple]:
"""
@ -189,8 +192,9 @@ class Backtesting:
data: Dict = {}
# Create dict with data
for pair, pair_data in processed.items():
pair_data.loc[:, 'buy'] = 0 # cleanup from previous run
pair_data.loc[:, 'sell'] = 0 # cleanup from previous run
if not pair_data.empty:
pair_data.loc[:, 'buy'] = 0 # cleanup if buy_signal is exist
pair_data.loc[:, 'sell'] = 0 # cleanup if sell_signal is exist
df_analyzed = self.strategy.advise_sell(
self.strategy.advise_buy(pair_data, {'pair': pair}), {'pair': pair})[headers].copy()
@ -214,6 +218,12 @@ class Backtesting:
"""
# Special handling if high or low hit STOP_LOSS or ROI
if sell.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS):
if trade.stop_loss > sell_row[HIGH_IDX]:
# our stoploss was already higher than candle high,
# possibly due to a cancelled trade exit.
# sell at open price.
return sell_row[OPEN_IDX]
# Set close_rate to stoploss
return trade.stop_loss
elif sell.sell_type == (SellType.ROI):
@ -239,7 +249,7 @@ class Backtesting:
# Use the maximum between close_rate and low as we
# cannot sell outside of a candle.
# Applies when a new ROI setting comes in place and the whole candle is above that.
return max(close_rate, sell_row[LOW_IDX])
return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX])
else:
# This should not be reached...
@ -250,12 +260,13 @@ class Backtesting:
def _get_sell_trade_entry(self, trade: LocalTrade, sell_row: Tuple) -> Optional[LocalTrade]:
sell = self.strategy.should_sell(trade, sell_row[OPEN_IDX], # type: ignore
sell_row[DATE_IDX], sell_row[BUY_IDX], sell_row[SELL_IDX],
sell_row[DATE_IDX].to_pydatetime(), sell_row[BUY_IDX],
sell_row[SELL_IDX],
low=sell_row[LOW_IDX], high=sell_row[HIGH_IDX])
if sell.sell_flag:
trade.close_date = sell_row[DATE_IDX]
trade.sell_reason = sell.sell_type.value
trade.close_date = sell_row[DATE_IDX].to_pydatetime()
trade.sell_reason = sell.sell_reason
trade_dur = int((trade.close_date_utc - trade.open_date_utc).total_seconds() // 60)
closerate = self._get_close_rate(sell_row, trade, sell, trade_dur)
@ -265,7 +276,8 @@ class Backtesting:
pair=trade.pair, trade=trade, order_type='limit', amount=trade.amount,
rate=closerate,
time_in_force=time_in_force,
sell_reason=sell.sell_type.value):
sell_reason=sell.sell_reason,
current_time=sell_row[DATE_IDX].to_pydatetime()):
return None
trade.close(closerate, show_msg=False)
@ -273,11 +285,9 @@ class Backtesting:
return None
def _enter_trade(self, pair: str, row: List, max_open_trades: int,
open_trade_count: int) -> Optional[LocalTrade]:
def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]:
try:
stake_amount = self.wallets.get_trade_stake_amount(
pair, max_open_trades - open_trade_count, None)
stake_amount = self.wallets.get_trade_stake_amount(pair, None)
except DependencyException:
return None
min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05)
@ -287,7 +297,7 @@ class Backtesting:
# Confirm trade entry:
if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)(
pair=pair, order_type=order_type, amount=stake_amount, rate=row[OPEN_IDX],
time_in_force=time_in_force):
time_in_force=time_in_force, current_time=row[DATE_IDX].to_pydatetime()):
return None
if stake_amount and (not min_stake_amount or stake_amount > min_stake_amount):
@ -295,7 +305,7 @@ class Backtesting:
trade = LocalTrade(
pair=pair,
open_rate=row[OPEN_IDX],
open_date=row[DATE_IDX],
open_date=row[DATE_IDX].to_pydatetime(),
stake_amount=stake_amount,
amount=round(stake_amount / row[OPEN_IDX], 8),
fee_open=self.fee,
@ -317,7 +327,7 @@ class Backtesting:
for trade in open_trades[pair]:
sell_row = data[pair][-1]
trade.close_date = sell_row[DATE_IDX]
trade.close_date = sell_row[DATE_IDX].to_pydatetime()
trade.sell_reason = SellType.FORCE_SELL.value
trade.close(sell_row[OPEN_IDX], show_msg=False)
LocalTrade.close_bt_trade(trade)
@ -327,10 +337,18 @@ class Backtesting:
trades.append(trade1)
return trades
def trade_slot_available(self, max_open_trades: int, open_trade_count: int) -> bool:
# Always allow trades when max_open_trades is enabled.
if max_open_trades <= 0 or open_trade_count < max_open_trades:
return True
# Rejected trade
self.rejected_trades += 1
return False
def backtest(self, processed: Dict,
start_date: datetime, end_date: datetime,
max_open_trades: int = 0, position_stacking: bool = False,
enable_protections: bool = False) -> DataFrame:
enable_protections: bool = False) -> Dict[str, Any]:
"""
Implement backtesting functionality
@ -349,12 +367,16 @@ class Backtesting:
trades: List[LocalTrade] = []
self.prepare_backtest(enable_protections)
# Update dataprovider cache
for pair, dataframe in processed.items():
self.dataprovider._set_cached_df(pair, self.timeframe, dataframe)
# Use dict of lists with data for performance
# (looping lists is a lot faster than pandas DataFrames)
data: Dict = self._get_ohlcv_as_lists(processed)
# Indexes per pair, so some pairs are allowed to have a missing start.
indexes: Dict = {}
indexes: Dict = defaultdict(int)
tmp = start_date + timedelta(minutes=self.timeframe_min)
open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
@ -365,11 +387,9 @@ class Backtesting:
open_trade_count_start = open_trade_count
for i, pair in enumerate(data):
if pair not in indexes:
indexes[pair] = 0
row_index = indexes[pair]
try:
row = data[pair][indexes[pair]]
row = data[pair][row_index]
except IndexError:
# missing Data for one pair at the end.
# Warnings for this are shown during data loading
@ -378,17 +398,23 @@ class Backtesting:
# Waits until the time-counter reaches the start of the data for this pair.
if row[DATE_IDX] > tmp:
continue
indexes[pair] += 1
row_index += 1
self.dataprovider._set_dataframe_max_index(row_index)
indexes[pair] = row_index
# without positionstacking, we can only have one open trade per pair.
# max_open_trades must be respected
# don't open on the last row
if ((position_stacking or len(open_trades[pair]) == 0)
and (max_open_trades <= 0 or open_trade_count_start < max_open_trades)
and tmp != end_date
and row[BUY_IDX] == 1 and row[SELL_IDX] != 1
and not PairLocks.is_pair_locked(pair, row[DATE_IDX])):
trade = self._enter_trade(pair, row, max_open_trades, open_trade_count_start)
if (
(position_stacking or len(open_trades[pair]) == 0)
and self.trade_slot_available(max_open_trades, open_trade_count_start)
and tmp != end_date
and row[BUY_IDX] == 1
and row[SELL_IDX] != 1
and not PairLocks.is_pair_locked(pair, row[DATE_IDX])
):
trade = self._enter_trade(pair, row)
if trade:
# TODO: hacky workaround to avoid opening > max_open_trades
# This emulates previous behaviour - not sure if this is correct
@ -420,7 +446,14 @@ class Backtesting:
trades += self.handle_left_open(open_trades, data=data)
self.wallets.update()
return trade_list_to_dataframe(trades)
results = trade_list_to_dataframe(trades)
return {
'results': results,
'config': self.strategy.config,
'locks': PairLocks.get_all_locks(),
'rejected_signals': self.rejected_trades,
'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
}
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any], timerange: TimeRange):
logger.info("Running backtesting for Strategy %s", strat.get_strategy_name())
@ -442,31 +475,32 @@ class Backtesting:
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = trim_dataframe(df, timerange)
min_date, max_date = history.get_timerange(preprocessed)
preprocessed = trim_dataframes(preprocessed, timerange, self.required_startup)
if not preprocessed:
raise OperationalException(
"No data left after adjusting for startup candles.")
min_date, max_date = history.get_timerange(preprocessed)
logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
f'({(max_date - min_date).days} days).')
# Execute backtest and store results
results = self.backtest(
processed=preprocessed,
start_date=min_date.datetime,
end_date=max_date.datetime,
start_date=min_date,
end_date=max_date,
max_open_trades=max_open_trades,
position_stacking=self.config.get('position_stacking', False),
enable_protections=self.config.get('enable_protections', False),
)
backtest_end_time = datetime.now(timezone.utc)
self.all_results[self.strategy.get_strategy_name()] = {
'results': results,
'config': self.strategy.config,
'locks': PairLocks.get_all_locks(),
'final_balance': self.wallets.get_total(self.strategy.config['stake_currency']),
results.update({
'backtest_start_time': int(backtest_start_time.timestamp()),
'backtest_end_time': int(backtest_end_time.timestamp()),
}
})
self.all_results[self.strategy.get_strategy_name()] = results
return min_date, max_date
def start(self) -> None:
@ -477,6 +511,7 @@ class Backtesting:
data: Dict[str, Any] = {}
data, timerange = self.load_bt_data()
logger.info("Dataload complete. Calculating indicators")
for strat in self.strategylist:
min_date, max_date = self.backtest_one_strategy(strat, data, timerange)

2
freqtrade/optimize/edge_cli.py
View File

@ -44,7 +44,7 @@ class EdgeCli:
'timerange') is None else str(self.config.get('timerange')))
def start(self) -> None:
result = self.edge.calculate()
result = self.edge.calculate(self.config['exchange']['pair_whitelist'])
if result:
print('') # blank line for readability
print(generate_edge_table(self.edge._cached_pairs))

275
freqtrade/optimize/hyperopt.py
View File

@ -4,33 +4,33 @@
This module contains the hyperopt logic
"""
import locale
import logging
import random
import warnings
from datetime import datetime
from datetime import datetime, timezone
from math import ceil
from operator import itemgetter
from pathlib import Path
from typing import Any, Dict, List, Optional
import progressbar
import rapidjson
from colorama import Fore, Style
from colorama import init as colorama_init
from joblib import Parallel, cpu_count, delayed, dump, load, wrap_non_picklable_objects
from pandas import DataFrame
from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN
from freqtrade.data.converter import trim_dataframe
from freqtrade.data.converter import trim_dataframes
from freqtrade.data.history import get_timerange
from freqtrade.misc import file_dump_json, plural
from freqtrade.optimize.backtesting import Backtesting
# Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
from freqtrade.optimize.hyperopt_auto import HyperOptAuto
from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401
from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401
from freqtrade.optimize.hyperopt_tools import HyperoptTools
from freqtrade.optimize.optimize_reports import generate_strategy_stats
from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver
from freqtrade.strategy import IStrategy
# Suppress scikit-learn FutureWarnings from skopt
@ -61,22 +61,33 @@ class Hyperopt:
hyperopt = Hyperopt(config)
hyperopt.start()
"""
custom_hyperopt: IHyperOpt
def __init__(self, config: Dict[str, Any]) -> None:
self.buy_space: List[Dimension] = []
self.sell_space: List[Dimension] = []
self.roi_space: List[Dimension] = []
self.stoploss_space: List[Dimension] = []
self.trailing_space: List[Dimension] = []
self.dimensions: List[Dimension] = []
self.config = config
self.backtesting = Backtesting(self.config)
self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
self.custom_hyperopt.__class__.strategy = self.backtesting.strategy
if not self.config.get('hyperopt'):
self.custom_hyperopt = HyperOptAuto(self.config)
else:
self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
self.backtesting._set_strategy(self.backtesting.strategylist[0])
self.custom_hyperopt.strategy = self.backtesting.strategy
self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config)
self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function
time_now = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
strategy = str(self.config['strategy'])
self.results_file = (self.config['user_data_dir'] /
'hyperopt_results' /
f'strategy_{strategy}_hyperopt_results_{time_now}.pickle')
self.results_file: Path = (self.config['user_data_dir'] / 'hyperopt_results' /
f'strategy_{strategy}_{time_now}.fthypt')
self.data_pickle_file = (self.config['user_data_dir'] /
'hyperopt_results' / 'hyperopt_tickerdata.pkl')
self.total_epochs = config.get('epochs', 0)
@ -86,9 +97,7 @@ class Hyperopt:
self.clean_hyperopt()
self.num_epochs_saved = 0
# Previous evaluations
self.epochs: List = []
self.current_best_epoch: Optional[Dict[str, Any]] = None
# Populate functions here (hasattr is slow so should not be run during "regular" operations)
if hasattr(self.custom_hyperopt, 'populate_indicators'):
@ -109,7 +118,7 @@ class Hyperopt:
self.max_open_trades = 0
self.position_stacking = self.config.get('position_stacking', False)
if self.has_space('sell'):
if HyperoptTools.has_space(self.config, 'sell'):
# Make sure use_sell_signal is enabled
if 'ask_strategy' not in self.config:
self.config['ask_strategy'] = {}
@ -135,9 +144,7 @@ class Hyperopt:
logger.info(f"Removing `{p}`.")
p.unlink()
def _get_params_dict(self, raw_params: List[Any]) -> Dict:
dimensions: List[Dimension] = self.dimensions
def _get_params_dict(self, dimensions: List[Dimension], raw_params: List[Any]) -> Dict:
# Ensure the number of dimensions match
# the number of parameters in the list.
@ -148,21 +155,24 @@ class Hyperopt:
# and the values are taken from the list of parameters.
return {d.name: v for d, v in zip(dimensions, raw_params)}
def _save_results(self) -> None:
def _save_result(self, epoch: Dict) -> None:
"""
Save hyperopt results to file
Store one line per epoch.
While not a valid json object - this allows appending easily.
:param epoch: result dictionary for this epoch.
"""
num_epochs = len(self.epochs)
if num_epochs > self.num_epochs_saved:
logger.debug(f"Saving {num_epochs} {plural(num_epochs, 'epoch')}.")
dump(self.epochs, self.results_file)
self.num_epochs_saved = num_epochs
logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
f"saved to '{self.results_file}'.")
# Store hyperopt filename
latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN)
file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)},
log=False)
with self.results_file.open('a') as f:
rapidjson.dump(epoch, f, default=str, number_mode=rapidjson.NM_NATIVE)
f.write("\n")
self.num_epochs_saved += 1
logger.debug(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
f"saved to '{self.results_file}'.")
# Store hyperopt filename
latest_filename = Path.joinpath(self.results_file.parent, LAST_BT_RESULT_FN)
file_dump_json(latest_filename, {'latest_hyperopt': str(self.results_file.name)},
log=False)
def _get_params_details(self, params: Dict) -> Dict:
"""
@ -170,18 +180,16 @@ class Hyperopt:
"""
result: Dict = {}
if self.has_space('buy'):
result['buy'] = {p.name: params.get(p.name)
for p in self.hyperopt_space('buy')}
if self.has_space('sell'):
result['sell'] = {p.name: params.get(p.name)
for p in self.hyperopt_space('sell')}
if self.has_space('roi'):
result['roi'] = self.custom_hyperopt.generate_roi_table(params)
if self.has_space('stoploss'):
result['stoploss'] = {p.name: params.get(p.name)
for p in self.hyperopt_space('stoploss')}
if self.has_space('trailing'):
if HyperoptTools.has_space(self.config, 'buy'):
result['buy'] = {p.name: params.get(p.name) for p in self.buy_space}
if HyperoptTools.has_space(self.config, 'sell'):
result['sell'] = {p.name: params.get(p.name) for p in self.sell_space}
if HyperoptTools.has_space(self.config, 'roi'):
result['roi'] = {str(k): v for k, v in
self.custom_hyperopt.generate_roi_table(params).items()}
if HyperoptTools.has_space(self.config, 'stoploss'):
result['stoploss'] = {p.name: params.get(p.name) for p in self.stoploss_space}
if HyperoptTools.has_space(self.config, 'trailing'):
result['trailing'] = self.custom_hyperopt.generate_trailing_params(params)
return result
@ -203,71 +211,58 @@ class Hyperopt:
)
self.hyperopt_table_header = 2
def has_space(self, space: str) -> bool:
def init_spaces(self):
"""
Tell if the space value is contained in the configuration
Assign the dimensions in the hyperoptimization space.
"""
# The 'trailing' space is not included in the 'default' set of spaces
if space == 'trailing':
return any(s in self.config['spaces'] for s in [space, 'all'])
else:
return any(s in self.config['spaces'] for s in [space, 'all', 'default'])
def hyperopt_space(self, space: Optional[str] = None) -> List[Dimension]:
"""
Return the dimensions in the hyperoptimization space.
:param space: Defines hyperspace to return dimensions for.
If None, then the self.has_space() will be used to return dimensions
for all hyperspaces used.
"""
spaces: List[Dimension] = []
if space == 'buy' or (space is None and self.has_space('buy')):
if HyperoptTools.has_space(self.config, 'buy'):
logger.debug("Hyperopt has 'buy' space")
spaces += self.custom_hyperopt.indicator_space()
self.buy_space = self.custom_hyperopt.indicator_space()
if space == 'sell' or (space is None and self.has_space('sell')):
if HyperoptTools.has_space(self.config, 'sell'):
logger.debug("Hyperopt has 'sell' space")
spaces += self.custom_hyperopt.sell_indicator_space()
self.sell_space = self.custom_hyperopt.sell_indicator_space()
if space == 'roi' or (space is None and self.has_space('roi')):
if HyperoptTools.has_space(self.config, 'roi'):
logger.debug("Hyperopt has 'roi' space")
spaces += self.custom_hyperopt.roi_space()
self.roi_space = self.custom_hyperopt.roi_space()
if space == 'stoploss' or (space is None and self.has_space('stoploss')):
if HyperoptTools.has_space(self.config, 'stoploss'):
logger.debug("Hyperopt has 'stoploss' space")
spaces += self.custom_hyperopt.stoploss_space()
self.stoploss_space = self.custom_hyperopt.stoploss_space()
if space == 'trailing' or (space is None and self.has_space('trailing')):
if HyperoptTools.has_space(self.config, 'trailing'):
logger.debug("Hyperopt has 'trailing' space")
spaces += self.custom_hyperopt.trailing_space()
return spaces
self.trailing_space = self.custom_hyperopt.trailing_space()
self.dimensions = (self.buy_space + self.sell_space + self.roi_space +
self.stoploss_space + self.trailing_space)
def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict:
"""
Used Optimize function. Called once per epoch to optimize whatever is configured.
Keep this function as optimized as possible!
"""
params_dict = self._get_params_dict(raw_params)
params_details = self._get_params_details(params_dict)
backtest_start_time = datetime.now(timezone.utc)
params_dict = self._get_params_dict(self.dimensions, raw_params)
if self.has_space('roi'):
# Apply parameters
if HyperoptTools.has_space(self.config, 'roi'):
self.backtesting.strategy.minimal_roi = ( # type: ignore
self.custom_hyperopt.generate_roi_table(params_dict))
if self.has_space('buy'):
if HyperoptTools.has_space(self.config, 'buy'):
self.backtesting.strategy.advise_buy = ( # type: ignore
self.custom_hyperopt.buy_strategy_generator(params_dict))
if self.has_space('sell'):
if HyperoptTools.has_space(self.config, 'sell'):
self.backtesting.strategy.advise_sell = ( # type: ignore
self.custom_hyperopt.sell_strategy_generator(params_dict))
if self.has_space('stoploss'):
if HyperoptTools.has_space(self.config, 'stoploss'):
self.backtesting.strategy.stoploss = params_dict['stoploss']
if self.has_space('trailing'):
if HyperoptTools.has_space(self.config, 'trailing'):
d = self.custom_hyperopt.generate_trailing_params(params_dict)
self.backtesting.strategy.trailing_stop = d['trailing_stop']
self.backtesting.strategy.trailing_stop_positive = d['trailing_stop_positive']
@ -276,30 +271,42 @@ class Hyperopt:
self.backtesting.strategy.trailing_only_offset_is_reached = \
d['trailing_only_offset_is_reached']
processed = load(self.data_pickle_file)
min_date, max_date = get_timerange(processed)
backtesting_results = self.backtesting.backtest(
with self.data_pickle_file.open('rb') as f:
processed = load(f, mmap_mode='r')
bt_results = self.backtesting.backtest(
processed=processed,
start_date=min_date.datetime,
end_date=max_date.datetime,
start_date=self.min_date,
end_date=self.max_date,
max_open_trades=self.max_open_trades,
position_stacking=self.position_stacking,
enable_protections=self.config.get('enable_protections', False),
)
return self._get_results_dict(backtesting_results, min_date, max_date,
params_dict, params_details,
backtest_end_time = datetime.now(timezone.utc)
bt_results.update({
'backtest_start_time': int(backtest_start_time.timestamp()),
'backtest_end_time': int(backtest_end_time.timestamp()),
})
return self._get_results_dict(bt_results, self.min_date, self.max_date,
params_dict,
processed=processed)
def _get_results_dict(self, backtesting_results, min_date, max_date,
params_dict, params_details, processed: Dict[str, DataFrame]):
results_metrics = self._calculate_results_metrics(backtesting_results)
results_explanation = self._format_results_explanation_string(results_metrics)
params_dict, processed: Dict[str, DataFrame]
) -> Dict[str, Any]:
params_details = self._get_params_details(params_dict)
trade_count = results_metrics['trade_count']
total_profit = results_metrics['total_profit']
strat_stats = generate_strategy_stats(
processed, self.backtesting.strategy.get_strategy_name(),
backtesting_results, min_date, max_date, market_change=0
)
results_explanation = HyperoptTools.format_results_explanation_string(
strat_stats, self.config['stake_currency'])
not_optimized = self.backtesting.strategy.get_params_dict()
trade_count = strat_stats['total_trades']
total_profit = strat_stats['profit_total']
# If this evaluation contains too short amount of trades to be
# interesting -- consider it as 'bad' (assigned max. loss value)
@ -307,50 +314,20 @@ class Hyperopt:
# path. We do not want to optimize 'hodl' strategies.
loss: float = MAX_LOSS
if trade_count >= self.config['hyperopt_min_trades']:
loss = self.calculate_loss(results=backtesting_results, trade_count=trade_count,
min_date=min_date.datetime, max_date=max_date.datetime,
loss = self.calculate_loss(results=backtesting_results['results'],
trade_count=trade_count,
min_date=min_date, max_date=max_date,
config=self.config, processed=processed)
return {
'loss': loss,
'params_dict': params_dict,
'params_details': params_details,
'results_metrics': results_metrics,
'params_not_optimized': not_optimized,
'results_metrics': strat_stats,
'results_explanation': results_explanation,
'total_profit': total_profit,
}
def _calculate_results_metrics(self, backtesting_results: DataFrame) -> Dict:
wins = len(backtesting_results[backtesting_results['profit_ratio'] > 0])
draws = len(backtesting_results[backtesting_results['profit_ratio'] == 0])
losses = len(backtesting_results[backtesting_results['profit_ratio'] < 0])
return {
'trade_count': len(backtesting_results.index),
'wins': wins,
'draws': draws,
'losses': losses,
'winsdrawslosses': f"{wins:>4} {draws:>4} {losses:>4}",
'avg_profit': backtesting_results['profit_ratio'].mean() * 100.0,
'median_profit': backtesting_results['profit_ratio'].median() * 100.0,
'total_profit': backtesting_results['profit_abs'].sum(),
'profit': backtesting_results['profit_ratio'].sum() * 100.0,
'duration': backtesting_results['trade_duration'].mean(),
}
def _format_results_explanation_string(self, results_metrics: Dict) -> str:
"""
Return the formatted results explanation in a string
"""
stake_cur = self.config['stake_currency']
return (f"{results_metrics['trade_count']:6d} trades. "
f"{results_metrics['wins']}/{results_metrics['draws']}"
f"/{results_metrics['losses']} Wins/Draws/Losses. "
f"Avg profit {results_metrics['avg_profit']: 6.2f}%. "
f"Median profit {results_metrics['median_profit']: 6.2f}%. "
f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} "
f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
f"Avg duration {results_metrics['duration']:5.1f} min."
).encode(locale.getpreferredencoding(), 'replace').decode('utf-8')
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer:
return Optimizer(
dimensions,
@ -369,24 +346,31 @@ class Hyperopt:
def _set_random_state(self, random_state: Optional[int]) -> int:
return random_state or random.randint(1, 2**16 - 1)
def start(self) -> None:
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
logger.info(f"Using optimizer random state: {self.random_state}")
self.hyperopt_table_header = -1
def prepare_hyperopt_data(self) -> None:
data, timerange = self.backtesting.load_bt_data()
logger.info("Dataload complete. Calculating indicators")
preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = trim_dataframe(df, timerange)
min_date, max_date = get_timerange(preprocessed)
processed = trim_dataframes(preprocessed, timerange, self.backtesting.required_startup)
logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
self.min_date, self.max_date = get_timerange(processed)
dump(preprocessed, self.data_pickle_file)
logger.info(f'Hyperopting with data from {self.min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {self.max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(self.max_date - self.min_date).days} days)..')
dump(processed, self.data_pickle_file)
def start(self) -> None:
self.random_state = self._set_random_state(self.config.get('hyperopt_random_state', None))
logger.info(f"Using optimizer random state: {self.random_state}")
self.hyperopt_table_header = -1
# Initialize spaces ...
self.init_spaces()
self.prepare_hyperopt_data()
# We don't need exchange instance anymore while running hyperopt
self.backtesting.exchange.close()
@ -394,15 +378,12 @@ class Hyperopt:
self.backtesting.exchange._api_async = None # type: ignore
# self.backtesting.exchange = None # type: ignore
self.backtesting.pairlists = None # type: ignore
self.backtesting.strategy.dp = None # type: ignore
IStrategy.dp = None # type: ignore
cpus = cpu_count()
logger.info(f"Found {cpus} CPU cores. Let's make them scream!")
config_jobs = self.config.get('hyperopt_jobs', -1)
logger.info(f'Number of parallel jobs set as: {config_jobs}')
self.dimensions: List[Dimension] = self.hyperopt_space()
self.opt = self.get_optimizer(self.dimensions, config_jobs)
if self.print_colorized:
@ -468,25 +449,21 @@ class Hyperopt:
if is_best:
self.current_best_loss = val['loss']
self.epochs.append(val)
self.current_best_epoch = val
# Save results after each best epoch and every 100 epochs
if is_best or current % 100 == 0:
self._save_results()
self._save_result(val)
pbar.update(current)
except KeyboardInterrupt:
print('User interrupted..')
self._save_results()
logger.info(f"{self.num_epochs_saved} {plural(self.num_epochs_saved, 'epoch')} "
f"saved to '{self.results_file}'.")
if self.epochs:
sorted_epochs = sorted(self.epochs, key=itemgetter('loss'))
best_epoch = sorted_epochs[0]
HyperoptTools.print_epoch_details(best_epoch, self.total_epochs, self.print_json)
if self.current_best_epoch:
HyperoptTools.print_epoch_details(self.current_best_epoch, self.total_epochs,
self.print_json)
else:
# This is printed when Ctrl+C is pressed quickly, before first epochs have
# a chance to be evaluated.

89
freqtrade/optimize/hyperopt_auto.py Normal file
View File

@ -0,0 +1,89 @@
"""
HyperOptAuto class.
This module implements a convenience auto-hyperopt class, which can be used together with strategies
that implement IHyperStrategy interface.
"""
from contextlib import suppress
from typing import Any, Callable, Dict, List
from pandas import DataFrame
with suppress(ImportError):
from skopt.space import Dimension
from freqtrade.optimize.hyperopt_interface import IHyperOpt
class HyperOptAuto(IHyperOpt):
"""
This class delegates functionality to Strategy(IHyperStrategy) and Strategy.HyperOpt classes.
Most of the time Strategy.HyperOpt class would only implement indicator_space and
sell_indicator_space methods, but other hyperopt methods can be overridden as well.
"""
def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable:
def populate_buy_trend(dataframe: DataFrame, metadata: dict):
for attr_name, attr in self.strategy.enumerate_parameters('buy'):
if attr.optimize:
# noinspection PyProtectedMember
attr.value = params[attr_name]
return self.strategy.populate_buy_trend(dataframe, metadata)
return populate_buy_trend
def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable:
def populate_sell_trend(dataframe: DataFrame, metadata: dict):
for attr_name, attr in self.strategy.enumerate_parameters('sell'):
if attr.optimize:
# noinspection PyProtectedMember
attr.value = params[attr_name]
return self.strategy.populate_sell_trend(dataframe, metadata)
return populate_sell_trend
def _get_func(self, name) -> Callable:
"""
Return a function defined in Strategy.HyperOpt class, or one defined in super() class.
:param name: function name.
:return: a requested function.
"""
hyperopt_cls = getattr(self.strategy, 'HyperOpt', None)
default_func = getattr(super(), name)
if hyperopt_cls:
return getattr(hyperopt_cls, name, default_func)
else:
return default_func
def _generate_indicator_space(self, category):
for attr_name, attr in self.strategy.enumerate_parameters(category):
if attr.optimize:
yield attr.get_space(attr_name)
def _get_indicator_space(self, category, fallback_method_name):
indicator_space = list(self._generate_indicator_space(category))
if len(indicator_space) > 0:
return indicator_space
else:
return self._get_func(fallback_method_name)()
def indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('buy', 'indicator_space')
def sell_indicator_space(self) -> List['Dimension']:
return self._get_indicator_space('sell', 'sell_indicator_space')
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
return self._get_func('generate_roi_table')(params)
def roi_space(self) -> List['Dimension']:
return self._get_func('roi_space')()
def stoploss_space(self) -> List['Dimension']:
return self._get_func('stoploss_space')()
def generate_trailing_params(self, params: Dict) -> Dict:
return self._get_func('generate_trailing_params')(params)
def trailing_space(self) -> List['Dimension']:
return self._get_func('trailing_space')()

57
freqtrade/optimize/hyperopt_interface.py
View File

@ -7,11 +7,12 @@ import math
from abc import ABC
from typing import Any, Callable, Dict, List
from skopt.space import Categorical, Dimension, Integer, Real
from skopt.space import Categorical, Dimension, Integer
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import round_dict
from freqtrade.optimize.space import SKDecimal
from freqtrade.strategy import IStrategy
@ -31,7 +32,7 @@ class IHyperOpt(ABC):
Defines the mandatory structure must follow any custom hyperopt
Class attributes you can use:
ticker_interval -> int: value of the ticker interval to use for the strategy
timeframe -> int: value of the timeframe to use for the strategy
"""
ticker_interval: str # DEPRECATED
timeframe: str
@ -44,36 +45,31 @@ class IHyperOpt(ABC):
IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED
IHyperOpt.timeframe = str(config['timeframe'])
@staticmethod
def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable:
"""
Create a buy strategy generator.
"""
raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy'))
@staticmethod
def sell_strategy_generator(params: Dict[str, Any]) -> Callable:
def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable:
"""
Create a sell strategy generator.
"""
raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell'))
@staticmethod
def indicator_space() -> List[Dimension]:
def indicator_space(self) -> List[Dimension]:
"""
Create an indicator space.
"""
raise OperationalException(_format_exception_message('indicator_space', 'buy'))
@staticmethod
def sell_indicator_space() -> List[Dimension]:
def sell_indicator_space(self) -> List[Dimension]:
"""
Create a sell indicator space.
"""
raise OperationalException(_format_exception_message('sell_indicator_space', 'sell'))
@staticmethod
def generate_roi_table(params: Dict) -> Dict[int, float]:
def generate_roi_table(self, params: Dict) -> Dict[int, float]:
"""
Create a ROI table.
@ -88,8 +84,7 @@ class IHyperOpt(ABC):
return roi_table
@staticmethod
def roi_space() -> List[Dimension]:
def roi_space(self) -> List[Dimension]:
"""
Create a ROI space.
@ -97,7 +92,7 @@ class IHyperOpt(ABC):
This method implements adaptive roi hyperspace with varied
ranges for parameters which automatically adapts to the
ticker interval used.
timeframe used.
It's used by Freqtrade by default, if no custom roi_space method is defined.
"""
@ -109,7 +104,7 @@ class IHyperOpt(ABC):
roi_t_alpha = 1.0
roi_p_alpha = 1.0
timeframe_min = timeframe_to_minutes(IHyperOpt.ticker_interval)
timeframe_min = timeframe_to_minutes(self.timeframe)
# We define here limits for the ROI space parameters automagically adapted to the
# timeframe used by the bot:
@ -119,7 +114,7 @@ class IHyperOpt(ABC):
# * 'roi_p' (limits for the ROI value steps) components are scaled logarithmically.
#
# The scaling is designed so that it maps exactly to the legacy Freqtrade roi_space()
# method for the 5m ticker interval.
# method for the 5m timeframe.
roi_t_scale = timeframe_min / 5
roi_p_scale = math.log1p(timeframe_min) / math.log1p(5)
roi_limits = {
@ -145,7 +140,7 @@ class IHyperOpt(ABC):
'roi_p2': roi_limits['roi_p2_min'],
'roi_p3': roi_limits['roi_p3_min'],
}
logger.info(f"Min roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}")
logger.info(f"Min roi table: {round_dict(self.generate_roi_table(p), 3)}")
p = {
'roi_t1': roi_limits['roi_t1_max'],
'roi_t2': roi_limits['roi_t2_max'],
@ -154,19 +149,21 @@ class IHyperOpt(ABC):
'roi_p2': roi_limits['roi_p2_max'],
'roi_p3': roi_limits['roi_p3_max'],
}
logger.info(f"Max roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}")
logger.info(f"Max roi table: {round_dict(self.generate_roi_table(p), 3)}")
return [
Integer(roi_limits['roi_t1_min'], roi_limits['roi_t1_max'], name='roi_t1'),
Integer(roi_limits['roi_t2_min'], roi_limits['roi_t2_max'], name='roi_t2'),
Integer(roi_limits['roi_t3_min'], roi_limits['roi_t3_max'], name='roi_t3'),
Real(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], name='roi_p1'),
Real(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], name='roi_p2'),
Real(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], name='roi_p3'),
SKDecimal(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], decimals=3,
name='roi_p1'),
SKDecimal(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], decimals=3,
name='roi_p2'),
SKDecimal(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], decimals=3,
name='roi_p3'),
]
@staticmethod
def stoploss_space() -> List[Dimension]:
def stoploss_space(self) -> List[Dimension]:
"""
Create a stoploss space.
@ -174,11 +171,10 @@ class IHyperOpt(ABC):
You may override it in your custom Hyperopt class.
"""
return [
Real(-0.35, -0.02, name='stoploss'),
SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'),
]
@staticmethod
def generate_trailing_params(params: Dict) -> Dict:
def generate_trailing_params(self, params: Dict) -> Dict:
"""
Create dict with trailing stop parameters.
"""
@ -190,8 +186,7 @@ class IHyperOpt(ABC):
'trailing_only_offset_is_reached': params['trailing_only_offset_is_reached'],
}
@staticmethod
def trailing_space() -> List[Dimension]:
def trailing_space(self) -> List[Dimension]:
"""
Create a trailing stoploss space.
@ -206,14 +201,14 @@ class IHyperOpt(ABC):
# other 'trailing' hyperspace parameters.
Categorical([True], name='trailing_stop'),
Real(0.01, 0.35, name='trailing_stop_positive'),
SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'),
# 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive',
# so this intermediate parameter is used as the value of the difference between
# them. The value of the 'trailing_stop_positive_offset' is constructed in the
# generate_trailing_params() method.
# This is similar to the hyperspace dimensions used for constructing the ROI tables.
Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'),
SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'),
Categorical([True, False], name='trailing_only_offset_is_reached'),
]

189
freqtrade/optimize/hyperopt_tools.py
View File

@ -1,19 +1,18 @@
import io
import locale
import logging
from collections import OrderedDict
from pathlib import Path
from pprint import pformat
from typing import Dict, List
from typing import Any, Dict, List
import rapidjson
import tabulate
from colorama import Fore, Style
from joblib import load
from pandas import isna, json_normalize
from freqtrade.exceptions import OperationalException
from freqtrade.misc import round_dict
from freqtrade.misc import round_coin_value, round_dict
logger = logging.getLogger(__name__)
@ -21,13 +20,38 @@ logger = logging.getLogger(__name__)
class HyperoptTools():
@staticmethod
def has_space(config: Dict[str, Any], space: str) -> bool:
"""
Tell if the space value is contained in the configuration
"""
# The 'trailing' space is not included in the 'default' set of spaces
if space == 'trailing':
return any(s in config['spaces'] for s in [space, 'all'])
else:
return any(s in config['spaces'] for s in [space, 'all', 'default'])
@staticmethod
def _read_results_pickle(results_file: Path) -> List:
"""
Read hyperopt results from pickle file
LEGACY method - new files are written as json and cannot be read with this method.
"""
from joblib import load
logger.info(f"Reading pickled epochs from '{results_file}'")
data = load(results_file)
return data
@staticmethod
def _read_results(results_file: Path) -> List:
"""
Read hyperopt results from file
"""
logger.info("Reading epochs from '%s'", results_file)
data = load(results_file)
import rapidjson
logger.info(f"Reading epochs from '{results_file}'")
with results_file.open('r') as f:
data = [rapidjson.loads(line) for line in f]
return data
@staticmethod
@ -37,7 +61,10 @@ class HyperoptTools():
"""
epochs: List = []
if results_file.is_file() and results_file.stat().st_size > 0:
epochs = HyperoptTools._read_results(results_file)
if results_file.suffix == '.pickle':
epochs = HyperoptTools._read_results_pickle(results_file)
else:
epochs = HyperoptTools._read_results(results_file)
# Detection of some old format, without 'is_best' field saved
if epochs[0].get('is_best') is None:
raise OperationalException(
@ -53,6 +80,7 @@ class HyperoptTools():
Display details of the hyperopt result
"""
params = results.get('params_details', {})
non_optimized = results.get('params_not_optimized', {})
# Default header string
if header_str is None:
@ -69,8 +97,10 @@ class HyperoptTools():
print(rapidjson.dumps(result_dict, default=str, number_mode=rapidjson.NM_NATIVE))
else:
HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:")
HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:")
HyperoptTools._params_pretty_print(params, 'buy', "Buy hyperspace params:",
non_optimized)
HyperoptTools._params_pretty_print(params, 'sell', "Sell hyperspace params:",
non_optimized)
HyperoptTools._params_pretty_print(params, 'roi', "ROI table:")
HyperoptTools._params_pretty_print(params, 'stoploss', "Stoploss:")
HyperoptTools._params_pretty_print(params, 'trailing', "Trailing stop:")
@ -96,12 +126,12 @@ class HyperoptTools():
result_dict.update(space_params)
@staticmethod
def _params_pretty_print(params, space: str, header: str) -> None:
if space in params:
def _params_pretty_print(params, space: str, header: str, non_optimized={}) -> None:
if space in params or space in non_optimized:
space_params = HyperoptTools._space_params(params, space, 5)
params_result = f"\n# {header}\n"
result = f"\n# {header}\n"
if space == 'stoploss':
params_result += f"stoploss = {space_params.get('stoploss')}"
result += f"stoploss = {space_params.get('stoploss')}"
elif space == 'roi':
# TODO: get rid of OrderedDict when support for python 3.6 will be
# dropped (dicts keep the order as the language feature)
@ -110,28 +140,64 @@ class HyperoptTools():
(str(k), v) for k, v in space_params.items()
),
default=str, indent=4, number_mode=rapidjson.NM_NATIVE)
params_result += f"minimal_roi = {minimal_roi_result}"
result += f"minimal_roi = {minimal_roi_result}"
elif space == 'trailing':
for k, v in space_params.items():
params_result += f'{k} = {v}\n'
result += f'{k} = {v}\n'
else:
params_result += f"{space}_params = {pformat(space_params, indent=4)}"
params_result = params_result.replace("}", "\n}").replace("{", "{\n ")
no_params = HyperoptTools._space_params(non_optimized, space, 5)
params_result = params_result.replace("\n", "\n ")
print(params_result)
result += f"{space}_params = {HyperoptTools._pprint(space_params, no_params)}"
result = result.replace("\n", "\n ")
print(result)
@staticmethod
def _space_params(params, space: str, r: int = None) -> Dict:
d = params[space]
# Round floats to `r` digits after the decimal point if requested
return round_dict(d, r) if r else d
d = params.get(space)
if d:
# Round floats to `r` digits after the decimal point if requested
return round_dict(d, r) if r else d
return {}
@staticmethod
def _pprint(params, non_optimized, indent: int = 4):
"""
Pretty-print hyperopt results (based on 2 dicts - with add. comment)
"""
p = params.copy()
p.update(non_optimized)
result = '{\n'
for k, param in p.items():
result += " " * indent + f'"{k}": '
result += f'"{param}",' if isinstance(param, str) else f'{param},'
if k in non_optimized:
result += " # value loaded from strategy"
result += "\n"
result += '}'
return result
@staticmethod
def is_best_loss(results, current_best_loss: float) -> bool:
return results['loss'] < current_best_loss
return bool(results['loss'] < current_best_loss)
@staticmethod
def format_results_explanation_string(results_metrics: Dict, stake_currency: str) -> str:
"""
Return the formatted results explanation in a string
"""
return (f"{results_metrics['total_trades']:6d} trades. "
f"{results_metrics['wins']}/{results_metrics['draws']}"
f"/{results_metrics['losses']} Wins/Draws/Losses. "
f"Avg profit {results_metrics['profit_mean'] * 100: 6.2f}%. "
f"Median profit {results_metrics['profit_median'] * 100: 6.2f}%. "
f"Total profit {results_metrics['profit_total_abs']: 11.8f} {stake_currency} "
f"({results_metrics['profit_total'] * 100: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). "
f"Avg duration {results_metrics['holding_avg']} min."
).encode(locale.getpreferredencoding(), 'replace').decode('utf-8')
@staticmethod
def _format_explanation_string(results, total_epochs) -> str:
@ -156,12 +222,27 @@ class HyperoptTools():
if 'results_metrics.winsdrawslosses' not in trials.columns:
# Ensure compatibility with older versions of hyperopt results
trials['results_metrics.winsdrawslosses'] = 'N/A'
legacy_mode = True
if 'results_metrics.total_trades' in trials:
legacy_mode = False
# New mode, using backtest result for metrics
trials['results_metrics.winsdrawslosses'] = trials.apply(
lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} "
f"{x['results_metrics.losses']:>4}", axis=1)
trials = trials[['Best', 'current_epoch', 'results_metrics.total_trades',
'results_metrics.winsdrawslosses',
'results_metrics.profit_mean', 'results_metrics.profit_total_abs',
'results_metrics.profit_total', 'results_metrics.holding_avg',
'loss', 'is_initial_point', 'is_best']]
else:
# Legacy mode
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.winsdrawslosses',
'results_metrics.avg_profit', 'results_metrics.total_profit',
'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']]
trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.winsdrawslosses',
'results_metrics.avg_profit', 'results_metrics.total_profit',
'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']]
trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit',
'Total profit', 'Profit', 'Avg duration', 'Objective',
'is_initial_point', 'is_best']
@ -171,26 +252,28 @@ class HyperoptTools():
trials.loc[trials['is_initial_point'] & trials['is_best'], 'Best'] = '* Best'
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
trials['Trades'] = trials['Trades'].astype(str)
perc_multi = 1 if legacy_mode else 100
trials['Epoch'] = trials['Epoch'].apply(
lambda x: '{}/{}'.format(str(x).rjust(len(str(total_epochs)), ' '), total_epochs)
)
trials['Avg profit'] = trials['Avg profit'].apply(
lambda x: '{:,.2f}%'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
lambda x: f'{x * perc_multi:,.2f}%'.rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
)
trials['Avg duration'] = trials['Avg duration'].apply(
lambda x: '{:,.1f} m'.format(x).rjust(7, ' ') if not isna(x) else "--".rjust(7, ' ')
lambda x: f'{x:,.1f} m'.rjust(7, ' ') if isinstance(x, float) else f"{x}"
if not isna(x) else "--".rjust(7, ' ')
)
trials['Objective'] = trials['Objective'].apply(
lambda x: '{:,.5f}'.format(x).rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
lambda x: f'{x:,.5f}'.rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ')
)
stake_currency = config['stake_currency']
trials['Profit'] = trials.apply(
lambda x: '{:,.8f} {} {}'.format(
x['Total profit'], config['stake_currency'],
'({:,.2f}%)'.format(x['Profit']).rjust(10, ' ')
).rjust(25+len(config['stake_currency']))
if x['Total profit'] != 0.0 else '--'.rjust(25+len(config['stake_currency'])),
lambda x: '{} {}'.format(
round_coin_value(x['Total profit'], stake_currency),
'({:,.2f}%)'.format(x['Profit'] * perc_multi).rjust(10, ' ')
).rjust(25+len(stake_currency))
if x['Total profit'] != 0.0 else '--'.rjust(25+len(stake_currency)),
axis=1
)
trials = trials.drop(columns=['Total profit'])
@ -251,11 +334,21 @@ class HyperoptTools():
trials['Best'] = ''
trials['Stake currency'] = config['stake_currency']
base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.avg_profit', 'results_metrics.median_profit',
'results_metrics.total_profit',
'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']
if 'results_metrics.total_trades' in trials:
base_metrics = ['Best', 'current_epoch', 'results_metrics.total_trades',
'results_metrics.profit_mean', 'results_metrics.profit_median',
'results_metrics.profit_total',
'Stake currency',
'results_metrics.profit_total_abs', 'results_metrics.holding_avg',
'loss', 'is_initial_point', 'is_best']
perc_multi = 100
else:
perc_multi = 1
base_metrics = ['Best', 'current_epoch', 'results_metrics.trade_count',
'results_metrics.avg_profit', 'results_metrics.median_profit',
'results_metrics.total_profit',
'Stake currency', 'results_metrics.profit', 'results_metrics.duration',
'loss', 'is_initial_point', 'is_best']
param_metrics = [("params_dict."+param) for param in results[0]['params_dict'].keys()]
trials = trials[base_metrics + param_metrics]
@ -272,21 +365,23 @@ class HyperoptTools():
trials.loc[trials['Total profit'] > 0, 'is_profit'] = True
trials['Epoch'] = trials['Epoch'].astype(str)
trials['Trades'] = trials['Trades'].astype(str)
trials['Median profit'] = trials['Median profit'] * perc_multi
trials['Total profit'] = trials['Total profit'].apply(
lambda x: '{:,.8f}'.format(x) if x != 0.0 else ""
lambda x: f'{x:,.8f}' if x != 0.0 else ""
)
trials['Profit'] = trials['Profit'].apply(
lambda x: '{:,.2f}'.format(x) if not isna(x) else ""
lambda x: f'{x:,.2f}' if not isna(x) else ""
)
trials['Avg profit'] = trials['Avg profit'].apply(
lambda x: '{:,.2f}%'.format(x) if not isna(x) else ""
lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else ""
)
trials['Avg duration'] = trials['Avg duration'].apply(
lambda x: '{:,.1f} m'.format(x) if not isna(x) else ""
lambda x: f'{x:,.1f} m' if isinstance(
x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else ""
)
trials['Objective'] = trials['Objective'].apply(
lambda x: '{:,.5f}'.format(x) if x != 100000 else ""
lambda x: f'{x:,.5f}' if x != 100000 else ""
)
trials = trials.drop(columns=['is_initial_point', 'is_best', 'is_profit'])

448
freqtrade/optimize/optimize_reports.py
View File

@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Dict, List, Union
from arrow import Arrow
from numpy import int64
from pandas import DataFrame
from tabulate import tabulate
@ -44,7 +43,7 @@ def _get_line_floatfmt(stake_currency: str) -> List[str]:
Generate floatformat (goes in line with _generate_result_line())
"""
return ['s', 'd', '.2f', '.2f', f'.{decimals_per_coin(stake_currency)}f',
'.2f', 'd', 'd', 'd', 'd']
'.2f', 'd', 's', 's']
def _get_line_header(first_column: str, stake_currency: str) -> List[str]:
@ -53,7 +52,17 @@ def _get_line_header(first_column: str, stake_currency: str) -> List[str]:
"""
return [first_column, 'Buys', 'Avg Profit %', 'Cum Profit %',
f'Tot Profit {stake_currency}', 'Tot Profit %', 'Avg Duration',
'Wins', 'Draws', 'Losses']
'Win Draw Loss Win%']
def _generate_wins_draws_losses(wins, draws, losses):
if wins > 0 and losses == 0:
wl_ratio = '100'
elif wins == 0:
wl_ratio = '0'
else:
wl_ratio = f'{100.0 / (wins + draws + losses) * wins:.1f}' if losses > 0 else '100'
return f'{wins:>4} {draws:>4} {losses:>4} {wl_ratio:>4}'
def _generate_result_line(result: DataFrame, starting_balance: int, first_column: str) -> Dict:
@ -110,6 +119,9 @@ def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, starting_b
tabular_data.append(_generate_result_line(result, starting_balance, pair))
# Sort by total profit %:
tabular_data = sorted(tabular_data, key=lambda k: k['profit_total_abs'], reverse=True)
# Append Total
tabular_data.append(_generate_result_line(results, starting_balance, 'TOTAL'))
return tabular_data
@ -150,7 +162,7 @@ def generate_sell_reason_stats(max_open_trades: int, results: DataFrame) -> List
return tabular_data
def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
def generate_strategy_comparison(all_results: Dict) -> List[Dict]:
"""
Generate summary per strategy
:param all_results: Dict of <Strategyname: DataFrame> containing results for all strategies
@ -162,6 +174,17 @@ def generate_strategy_metrics(all_results: Dict) -> List[Dict]:
tabular_data.append(_generate_result_line(
results['results'], results['config']['dry_run_wallet'], strategy)
)
try:
max_drawdown_per, _, _, _, _ = calculate_max_drawdown(results['results'],
value_col='profit_ratio')
max_drawdown_abs, _, _, _, _ = calculate_max_drawdown(results['results'],
value_col='profit_abs')
except ValueError:
max_drawdown_per = 0
max_drawdown_abs = 0
tabular_data[-1]['max_drawdown_per'] = round(max_drawdown_per * 100, 2)
tabular_data[-1]['max_drawdown_abs'] = \
round_coin_value(max_drawdown_abs, results['config']['stake_currency'], False)
return tabular_data
@ -191,7 +214,40 @@ def generate_edge_table(results: dict) -> str:
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right") # type: ignore
def generate_trading_stats(results: DataFrame) -> Dict[str, Any]:
""" Generate overall trade statistics """
if len(results) == 0:
return {
'wins': 0,
'losses': 0,
'draws': 0,
'holding_avg': timedelta(),
'winner_holding_avg': timedelta(),
'loser_holding_avg': timedelta(),
}
winning_trades = results.loc[results['profit_ratio'] > 0]
draw_trades = results.loc[results['profit_ratio'] == 0]
losing_trades = results.loc[results['profit_ratio'] < 0]
zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) &
(results['sell_reason'] == 'trailing_stop_loss')])
return {
'wins': len(winning_trades),
'losses': len(losing_trades),
'draws': len(draw_trades),
'holding_avg': (timedelta(minutes=round(results['trade_duration'].mean()))
if not results.empty else timedelta()),
'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
if not winning_trades.empty else timedelta()),
'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
if not losing_trades.empty else timedelta()),
'zero_duration_trades': zero_duration_trades,
}
def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
""" Generate daily statistics """
if len(results) == 0:
return {
'backtest_best_day': 0,
@ -201,8 +257,6 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
'winning_days': 0,
'draw_days': 0,
'losing_days': 0,
'winner_holding_avg': timedelta(),
'loser_holding_avg': timedelta(),
}
daily_profit_rel = results.resample('1d', on='close_date')['profit_ratio'].sum()
daily_profit = results.resample('1d', on='close_date')['profit_abs'].sum().round(10)
@ -214,9 +268,6 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
draw_days = sum(daily_profit == 0)
losing_days = sum(daily_profit < 0)
winning_trades = results.loc[results['profit_ratio'] > 0]
losing_trades = results.loc[results['profit_ratio'] < 0]
return {
'backtest_best_day': best_rel,
'backtest_worst_day': worst_rel,
@ -225,16 +276,152 @@ def generate_daily_stats(results: DataFrame) -> Dict[str, Any]:
'winning_days': winning_days,
'draw_days': draw_days,
'losing_days': losing_days,
'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean()))
if not winning_trades.empty else timedelta()),
'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean()))
if not losing_trades.empty else timedelta()),
}
def generate_strategy_stats(btdata: Dict[str, DataFrame],
strategy: str,
content: Dict[str, Any],
min_date: datetime, max_date: datetime,
market_change: float
) -> Dict[str, Any]:
"""
:param btdata: Backtest data
:param strategy: Strategy name
:param content: Backtest result data in the format:
{'results: results, 'config: config}}.
:param min_date: Backtest start date
:param max_date: Backtest end date
:param market_change: float indicating the market change
:return: Dictionary containing results per strategy and a stratgy summary.
"""
results: Dict[str, DataFrame] = content['results']
if not isinstance(results, DataFrame):
return {}
config = content['config']
max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
starting_balance = config['dry_run_wallet']
stake_currency = config['stake_currency']
pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
starting_balance=starting_balance,
results=results, skip_nan=False)
sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades,
results=results)
left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
starting_balance=starting_balance,
results=results.loc[results['is_open']],
skip_nan=True)
daily_stats = generate_daily_stats(results)
trade_stats = generate_trading_stats(results)
best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
backtest_days = (max_date - min_date).days
strat_stats = {
'trades': results.to_dict(orient='records'),
'locks': [lock.to_json() for lock in content['locks']],
'best_pair': best_pair,
'worst_pair': worst_pair,
'results_per_pair': pair_results,
'sell_reason_summary': sell_reason_stats,
'left_open_trades': left_open_results,
'total_trades': len(results),
'total_volume': float(results['stake_amount'].sum()),
'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0,
'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0,
'profit_median': results['profit_ratio'].median() if len(results) > 0 else 0,
'profit_total': results['profit_abs'].sum() / starting_balance,
'profit_total_abs': results['profit_abs'].sum(),
'backtest_start': min_date.strftime(DATETIME_PRINT_FORMAT),
'backtest_start_ts': int(min_date.timestamp() * 1000),
'backtest_end': max_date.strftime(DATETIME_PRINT_FORMAT),
'backtest_end_ts': int(max_date.timestamp() * 1000),
'backtest_days': backtest_days,
'backtest_run_start_ts': content['backtest_start_time'],
'backtest_run_end_ts': content['backtest_end_time'],
'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
'market_change': market_change,
'pairlist': list(btdata.keys()),
'stake_amount': config['stake_amount'],
'stake_currency': config['stake_currency'],
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
'starting_balance': starting_balance,
'dry_run_wallet': starting_balance,
'final_balance': content['final_balance'],
'rejected_signals': content['rejected_signals'],
'max_open_trades': max_open_trades,
'max_open_trades_setting': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'timeframe': config['timeframe'],
'timerange': config.get('timerange', ''),
'enable_protections': config.get('enable_protections', False),
'strategy_name': strategy,
# Parameters relevant for backtesting
'stoploss': config['stoploss'],
'trailing_stop': config.get('trailing_stop', False),
'trailing_stop_positive': config.get('trailing_stop_positive'),
'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0),
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
'use_custom_stoploss': config.get('use_custom_stoploss', False),
'minimal_roi': config['minimal_roi'],
'use_sell_signal': config['ask_strategy']['use_sell_signal'],
'sell_profit_only': config['ask_strategy']['sell_profit_only'],
'sell_profit_offset': config['ask_strategy']['sell_profit_offset'],
'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
**daily_stats,
**trade_stats
}
try:
max_drawdown, _, _, _, _ = calculate_max_drawdown(
results, value_col='profit_ratio')
drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown(
results, value_col='profit_abs')
strat_stats.update({
'max_drawdown': max_drawdown,
'max_drawdown_abs': drawdown_abs,
'drawdown_start': drawdown_start.strftime(DATETIME_PRINT_FORMAT),
'drawdown_start_ts': drawdown_start.timestamp() * 1000,
'drawdown_end': drawdown_end.strftime(DATETIME_PRINT_FORMAT),
'drawdown_end_ts': drawdown_end.timestamp() * 1000,
'max_drawdown_low': low_val,
'max_drawdown_high': high_val,
})
csum_min, csum_max = calculate_csum(results, starting_balance)
strat_stats.update({
'csum_min': csum_min,
'csum_max': csum_max
})
except ValueError:
strat_stats.update({
'max_drawdown': 0.0,
'max_drawdown_abs': 0.0,
'max_drawdown_low': 0.0,
'max_drawdown_high': 0.0,
'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc),
'drawdown_start_ts': 0,
'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc),
'drawdown_end_ts': 0,
'csum_min': 0,
'csum_max': 0
})
return strat_stats
def generate_backtest_stats(btdata: Dict[str, DataFrame],
all_results: Dict[str, Dict[str, Union[DataFrame, Dict]]],
min_date: Arrow, max_date: Arrow
min_date: datetime, max_date: datetime
) -> Dict[str, Any]:
"""
:param btdata: Backtest data
@ -242,132 +429,17 @@ def generate_backtest_stats(btdata: Dict[str, DataFrame],
{ Strategy: {'results: results, 'config: config}}.
:param min_date: Backtest start date
:param max_date: Backtest end date
:return:
Dictionary containing results per strategy and a stratgy summary.
:return: Dictionary containing results per strategy and a stratgy summary.
"""
result: Dict[str, Any] = {'strategy': {}}
market_change = calculate_market_change(btdata, 'close')
for strategy, content in all_results.items():
results: Dict[str, DataFrame] = content['results']
if not isinstance(results, DataFrame):
continue
config = content['config']
max_open_trades = min(config['max_open_trades'], len(btdata.keys()))
starting_balance = config['dry_run_wallet']
stake_currency = config['stake_currency']
pair_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
starting_balance=starting_balance,
results=results, skip_nan=False)
sell_reason_stats = generate_sell_reason_stats(max_open_trades=max_open_trades,
results=results)
left_open_results = generate_pair_metrics(btdata, stake_currency=stake_currency,
starting_balance=starting_balance,
results=results.loc[results['is_open']],
skip_nan=True)
daily_stats = generate_daily_stats(results)
best_pair = max([pair for pair in pair_results if pair['key'] != 'TOTAL'],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
worst_pair = min([pair for pair in pair_results if pair['key'] != 'TOTAL'],
key=lambda x: x['profit_sum']) if len(pair_results) > 1 else None
results['open_timestamp'] = results['open_date'].astype(int64) // 1e6
results['close_timestamp'] = results['close_date'].astype(int64) // 1e6
backtest_days = (max_date - min_date).days
strat_stats = {
'trades': results.to_dict(orient='records'),
'locks': [lock.to_json() for lock in content['locks']],
'best_pair': best_pair,
'worst_pair': worst_pair,
'results_per_pair': pair_results,
'sell_reason_summary': sell_reason_stats,
'left_open_trades': left_open_results,
'total_trades': len(results),
'total_volume': float(results['stake_amount'].sum()),
'avg_stake_amount': results['stake_amount'].mean() if len(results) > 0 else 0,
'profit_mean': results['profit_ratio'].mean() if len(results) > 0 else 0,
'profit_total': results['profit_abs'].sum() / starting_balance,
'profit_total_abs': results['profit_abs'].sum(),
'backtest_start': min_date.datetime,
'backtest_start_ts': min_date.int_timestamp * 1000,
'backtest_end': max_date.datetime,
'backtest_end_ts': max_date.int_timestamp * 1000,
'backtest_days': backtest_days,
'backtest_run_start_ts': content['backtest_start_time'],
'backtest_run_end_ts': content['backtest_end_time'],
'trades_per_day': round(len(results) / backtest_days, 2) if backtest_days > 0 else 0,
'market_change': market_change,
'pairlist': list(btdata.keys()),
'stake_amount': config['stake_amount'],
'stake_currency': config['stake_currency'],
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
'starting_balance': starting_balance,
'dry_run_wallet': starting_balance,
'final_balance': content['final_balance'],
'max_open_trades': max_open_trades,
'max_open_trades_setting': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'timeframe': config['timeframe'],
'timerange': config.get('timerange', ''),
'enable_protections': config.get('enable_protections', False),
'strategy_name': strategy,
# Parameters relevant for backtesting
'stoploss': config['stoploss'],
'trailing_stop': config.get('trailing_stop', False),
'trailing_stop_positive': config.get('trailing_stop_positive'),
'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset', 0.0),
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached', False),
'use_custom_stoploss': config.get('use_custom_stoploss', False),
'minimal_roi': config['minimal_roi'],
'use_sell_signal': config['ask_strategy']['use_sell_signal'],
'sell_profit_only': config['ask_strategy']['sell_profit_only'],
'sell_profit_offset': config['ask_strategy']['sell_profit_offset'],
'ignore_roi_if_buy_signal': config['ask_strategy']['ignore_roi_if_buy_signal'],
**daily_stats,
}
strat_stats = generate_strategy_stats(btdata, strategy, content,
min_date, max_date, market_change=market_change)
result['strategy'][strategy] = strat_stats
try:
max_drawdown, _, _, _, _ = calculate_max_drawdown(
results, value_col='profit_ratio')
drawdown_abs, drawdown_start, drawdown_end, high_val, low_val = calculate_max_drawdown(
results, value_col='profit_abs')
strat_stats.update({
'max_drawdown': max_drawdown,
'max_drawdown_abs': drawdown_abs,
'drawdown_start': drawdown_start,
'drawdown_start_ts': drawdown_start.timestamp() * 1000,
'drawdown_end': drawdown_end,
'drawdown_end_ts': drawdown_end.timestamp() * 1000,
'max_drawdown_low': low_val,
'max_drawdown_high': high_val,
})
csum_min, csum_max = calculate_csum(results, starting_balance)
strat_stats.update({
'csum_min': csum_min,
'csum_max': csum_max
})
except ValueError:
strat_stats.update({
'max_drawdown': 0.0,
'max_drawdown_abs': 0.0,
'max_drawdown_low': 0.0,
'max_drawdown_high': 0.0,
'drawdown_start': datetime(1970, 1, 1, tzinfo=timezone.utc),
'drawdown_start_ts': 0,
'drawdown_end': datetime(1970, 1, 1, tzinfo=timezone.utc),
'drawdown_end_ts': 0,
'csum_min': 0,
'csum_max': 0
})
strategy_results = generate_strategy_metrics(all_results=all_results)
strategy_results = generate_strategy_comparison(all_results=all_results)
result['strategy_comparison'] = strategy_results
@ -390,7 +462,8 @@ def text_table_bt_results(pair_results: List[Dict[str, Any]], stake_currency: st
floatfmt = _get_line_floatfmt(stake_currency)
output = [[
t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses']
t['profit_total_pct'], t['duration_avg'],
_generate_wins_draws_losses(t['wins'], t['draws'], t['losses'])
] for t in pair_results]
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(output, headers=headers,
@ -407,9 +480,7 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
headers = [
'Sell Reason',
'Sells',
'Wins',
'Draws',
'Losses',
'Win Draws Loss Win%',
'Avg Profit %',
'Cum Profit %',
f'Tot Profit {stake_currency}',
@ -417,7 +488,8 @@ def text_table_sell_reason(sell_reason_stats: List[Dict[str, Any]], stake_curren
]
output = [[
t['sell_reason'], t['trades'], t['wins'], t['draws'], t['losses'],
t['sell_reason'], t['trades'],
_generate_wins_draws_losses(t['wins'], t['draws'], t['losses']),
t['profit_mean_pct'], t['profit_sum_pct'],
round_coin_value(t['profit_total_abs'], stake_currency, False),
t['profit_total_pct'],
@ -435,11 +507,22 @@ def text_table_strategy(strategy_results, stake_currency: str) -> str:
"""
floatfmt = _get_line_floatfmt(stake_currency)
headers = _get_line_header('Strategy', stake_currency)
# _get_line_header() is also used for per-pair summary. Per-pair drawdown is mostly useless
# therefore we slip this column in only for strategy summary here.
headers.append('Drawdown')
# Align drawdown string on the center two space separator.
drawdown = [f'{t["max_drawdown_per"]:.2f}' for t in strategy_results]
dd_pad_abs = max([len(t['max_drawdown_abs']) for t in strategy_results])
dd_pad_per = max([len(dd) for dd in drawdown])
drawdown = [f'{t["max_drawdown_abs"]:>{dd_pad_abs}} {stake_currency} {dd:>{dd_pad_per}}%'
for t, dd in zip(strategy_results, drawdown)]
output = [[
t['key'], t['trades'], t['profit_mean_pct'], t['profit_sum_pct'], t['profit_total_abs'],
t['profit_total_pct'], t['duration_avg'], t['wins'], t['draws'], t['losses']
] for t in strategy_results]
t['profit_total_pct'], t['duration_avg'],
_generate_wins_draws_losses(t['wins'], t['draws'], t['losses']), drawdown]
for t, drawdown in zip(strategy_results, drawdown)]
# Ignore type as floatfmt does allow tuples but mypy does not know that
return tabulate(output, headers=headers,
floatfmt=floatfmt, tablefmt="orgtbl", stralign="right")
@ -449,9 +532,21 @@ def text_table_add_metrics(strat_results: Dict) -> str:
if len(strat_results['trades']) > 0:
best_trade = max(strat_results['trades'], key=lambda x: x['profit_ratio'])
worst_trade = min(strat_results['trades'], key=lambda x: x['profit_ratio'])
# Newly added fields should be ignored if they are missing in strat_results. hyperopt-show
# command stores these results and newer version of freqtrade must be able to handle old
# results with missing new fields.
zero_duration_trades = '--'
if 'zero_duration_trades' in strat_results:
zero_duration_trades_per = \
100.0 / strat_results['total_trades'] * strat_results['zero_duration_trades']
zero_duration_trades = f'{zero_duration_trades_per:.2f}% ' \
f'({strat_results["zero_duration_trades"]})'
metrics = [
('Backtesting from', strat_results['backtest_start'].strftime(DATETIME_PRINT_FORMAT)),
('Backtesting to', strat_results['backtest_end'].strftime(DATETIME_PRINT_FORMAT)),
('Backtesting from', strat_results['backtest_start']),
('Backtesting to', strat_results['backtest_end']),
('Max open trades', strat_results['max_open_trades']),
('', ''), # Empty line to improve readability
('Total trades', strat_results['total_trades']),
@ -461,13 +556,12 @@ def text_table_add_metrics(strat_results: Dict) -> str:
strat_results['stake_currency'])),
('Absolute profit ', round_coin_value(strat_results['profit_total_abs'],
strat_results['stake_currency'])),
('Total profit %', f"{round(strat_results['profit_total'] * 100, 2)}%"),
('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"),
('Trades per day', strat_results['trades_per_day']),
('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'],
strat_results['stake_currency'])),
('Total trade volume', round_coin_value(strat_results['total_volume'],
strat_results['stake_currency'])),
('', ''), # Empty line to improve readability
('Best Pair', f"{strat_results['best_pair']['key']} "
f"{round(strat_results['best_pair']['profit_sum_pct'], 2)}%"),
@ -485,6 +579,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
f"{strat_results['draw_days']} / {strat_results['losing_days']}"),
('Avg. Duration Winners', f"{strat_results['winner_holding_avg']}"),
('Avg. Duration Loser', f"{strat_results['loser_holding_avg']}"),
('Zero Duration Trades', zero_duration_trades),
('Rejected Buy signals', strat_results.get('rejected_signals', 'N/A')),
('', ''), # Empty line to improve readability
('Min balance', round_coin_value(strat_results['csum_min'],
@ -499,8 +595,8 @@ def text_table_add_metrics(strat_results: Dict) -> str:
strat_results['stake_currency'])),
('Drawdown low', round_coin_value(strat_results['max_drawdown_low'],
strat_results['stake_currency'])),
('Drawdown Start', strat_results['drawdown_start'].strftime(DATETIME_PRINT_FORMAT)),
('Drawdown End', strat_results['drawdown_end'].strftime(DATETIME_PRINT_FORMAT)),
('Drawdown Start', strat_results['drawdown_start']),
('Drawdown End', strat_results['drawdown_end']),
('Market change', f"{round(strat_results['market_change'] * 100, 2)}%"),
]
@ -519,37 +615,43 @@ def text_table_add_metrics(strat_results: Dict) -> str:
return message
def show_backtest_result(strategy: str, results: Dict[str, Any], stake_currency: str):
"""
Print results for one strategy
"""
# Print results
print(f"Result for strategy {strategy}")
table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency)
if isinstance(table, str):
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
stake_currency=stake_currency)
if isinstance(table, str) and len(table) > 0:
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
if isinstance(table, str) and len(table) > 0:
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_add_metrics(results)
if isinstance(table, str) and len(table) > 0:
print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
print(table)
if isinstance(table, str) and len(table) > 0:
print('=' * len(table.splitlines()[0]))
print()
def show_backtest_results(config: Dict, backtest_stats: Dict):
stake_currency = config['stake_currency']
for strategy, results in backtest_stats['strategy'].items():
# Print results
print(f"Result for strategy {strategy}")
table = text_table_bt_results(results['results_per_pair'], stake_currency=stake_currency)
if isinstance(table, str):
print(' BACKTESTING REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_sell_reason(sell_reason_stats=results['sell_reason_summary'],
stake_currency=stake_currency)
if isinstance(table, str) and len(table) > 0:
print(' SELL REASON STATS '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_bt_results(results['left_open_trades'], stake_currency=stake_currency)
if isinstance(table, str) and len(table) > 0:
print(' LEFT OPEN TRADES REPORT '.center(len(table.splitlines()[0]), '='))
print(table)
table = text_table_add_metrics(results)
if isinstance(table, str) and len(table) > 0:
print(' SUMMARY METRICS '.center(len(table.splitlines()[0]), '='))
print(table)
if isinstance(table, str) and len(table) > 0:
print('=' * len(table.splitlines()[0]))
print()
show_backtest_result(strategy, results, stake_currency)
if len(backtest_stats['strategy']) > 1:
# Print Strategy summary table

4
freqtrade/optimize/space/__init__.py Normal file
View File

@ -0,0 +1,4 @@
# flake8: noqa: F401
from skopt.space import Categorical, Dimension, Integer, Real
from .decimalspace import SKDecimal

33
freqtrade/optimize/space/decimalspace.py Normal file
View File

@ -0,0 +1,33 @@
import numpy as np
from skopt.space import Integer
class SKDecimal(Integer):
def __init__(self, low, high, decimals=3, prior="uniform", base=10, transform=None,
name=None, dtype=np.int64):
self.decimals = decimals
_low = int(low * pow(10, self.decimals))
_high = int(high * pow(10, self.decimals))
# trunc to precision to avoid points out of space
self.low_orig = round(_low * pow(0.1, self.decimals), self.decimals)
self.high_orig = round(_high * pow(0.1, self.decimals), self.decimals)
super().__init__(_low, _high, prior, base, transform, name, dtype)
def __repr__(self):
return "Decimal(low={}, high={}, decimals={}, prior='{}', transform='{}')".format(
self.low_orig, self.high_orig, self.decimals, self.prior, self.transform_)
def __contains__(self, point):
if isinstance(point, list):
point = np.array(point)
return self.low_orig <= point <= self.high_orig
def transform(self, Xt):
aa = [int(x * pow(10, self.decimals)) for x in Xt]
return super().transform(aa)
def inverse_transform(self, Xt):
res = super().inverse_transform(Xt)
return [round(x * pow(0.1, self.decimals), self.decimals) for x in res]

32
freqtrade/persistence/migrations.py
View File

@ -123,6 +123,27 @@ def migrate_open_orders_to_trades(engine):
""")
def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List):
# Schema migration necessary
engine.execute(f"alter table orders rename to {table_back_name}")
# drop indexes on backup table
for index in inspector.get_indexes(table_back_name):
engine.execute(f"drop index {index['name']}")
# let SQLAlchemy create the schema as required
decl_base.metadata.create_all(engine)
engine.execute(f"""
insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status,
symbol, order_type, side, price, amount, filled, average, remaining, cost, order_date,
order_filled_date, order_update_date)
select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status,
symbol, order_type, side, price, amount, filled, null average, remaining, cost, order_date,
order_filled_date, order_update_date
from {table_back_name}
""")
def check_migrate(engine, decl_base, previous_tables) -> None:
"""
Checks if migration is necessary and migrates if necessary
@ -145,6 +166,11 @@ def check_migrate(engine, decl_base, previous_tables) -> None:
logger.info('Moving open orders to Orders table.')
migrate_open_orders_to_trades(engine)
else:
pass
# Empty for now - as there is only one iteration of the orders table so far.
# table_back_name = get_backup_name(tabs, 'orders_bak')
cols_order = inspector.get_columns('orders')
if not has_column(cols_order, 'average'):
tabs = get_table_names_for_table(inspector, 'orders')
# Empty for now - as there is only one iteration of the orders table so far.
table_back_name = get_backup_name(tabs, 'orders_bak')
migrate_orders_table(decl_base, inspector, engine, table_back_name, cols)

269
freqtrade/persistence/models.py
View File

@ -6,7 +6,6 @@ from datetime import datetime, timezone
from decimal import Decimal
from typing import Any, Dict, List, Optional
import arrow
from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String,
create_engine, desc, func, inspect)
from sqlalchemy.exc import NoSuchModuleError
@ -59,13 +58,10 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None:
# https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope
# Scoped sessions proxy requests to the appropriate thread-local session.
# We should use the scoped_session object - not a seperately initialized version
Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
Trade.query = Trade.session.query_property()
# Copy session attributes to order object too
Order.session = Trade.session
Order.query = Order.session.query_property()
PairLock.session = Trade.session
PairLock.query = PairLock.session.query_property()
Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
Trade.query = Trade._session.query_property()
Order.query = Trade._session.query_property()
PairLock.query = Trade._session.query_property()
previous_tables = inspect(engine).get_table_names()
_DECL_BASE.metadata.create_all(engine)
@ -81,7 +77,7 @@ def cleanup_db() -> None:
Flushes all pending operations to disk.
:return: None
"""
Trade.session.flush()
Trade.query.session.flush()
def clean_dry_run_db() -> None:
@ -116,16 +112,17 @@ class Order(_DECL_BASE):
trade = relationship("Trade", back_populates="orders")
ft_order_side = Column(String, nullable=False)
ft_pair = Column(String, nullable=False)
ft_order_side = Column(String(25), nullable=False)
ft_pair = Column(String(25), nullable=False)
ft_is_open = Column(Boolean, nullable=False, default=True, index=True)
order_id = Column(String, nullable=False, index=True)
status = Column(String, nullable=True)
symbol = Column(String, nullable=True)
order_type = Column(String, nullable=True)
side = Column(String, nullable=True)
order_id = Column(String(255), nullable=False, index=True)
status = Column(String(255), nullable=True)
symbol = Column(String(25), nullable=True)
order_type = Column(String(50), nullable=True)
side = Column(String(25), nullable=True)
price = Column(Float, nullable=True)
average = Column(Float, nullable=True)
amount = Column(Float, nullable=True)
filled = Column(Float, nullable=True)
remaining = Column(Float, nullable=True)
@ -154,6 +151,7 @@ class Order(_DECL_BASE):
self.price = order.get('price', self.price)
self.amount = order.get('amount', self.amount)
self.filled = order.get('filled', self.filled)
self.average = order.get('average', self.average)
self.remaining = order.get('remaining', self.remaining)
self.cost = order.get('cost', self.cost)
if 'timestamp' in order and order['timestamp'] is not None:
@ -163,8 +161,8 @@ class Order(_DECL_BASE):
if self.status in ('closed', 'canceled', 'cancelled'):
self.ft_is_open = False
if order.get('filled', 0) > 0:
self.order_filled_date = arrow.utcnow().datetime
self.order_update_date = arrow.utcnow().datetime
self.order_filled_date = datetime.now(timezone.utc)
self.order_update_date = datetime.now(timezone.utc)
@staticmethod
def update_orders(orders: List['Order'], order: Dict[str, Any]):
@ -297,15 +295,12 @@ class LocalTrade():
'fee_close_cost': self.fee_close_cost,
'fee_close_currency': self.fee_close_currency,
'open_date_hum': arrow.get(self.open_date).humanize(),
'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT),
'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
'open_rate': self.open_rate,
'open_rate_requested': self.open_rate_requested,
'open_trade_value': round(self.open_trade_value, 8),
'close_date_hum': (arrow.get(self.close_date).humanize()
if self.close_date else None),
'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT)
if self.close_date else None),
'close_timestamp': int(self.close_date.replace(
@ -554,6 +549,8 @@ class LocalTrade():
rate=(rate or self.close_rate),
fee=(fee or self.fee_close)
)
if self.open_trade_value == 0.0:
return 0.0
profit_ratio = (close_trade_value / self.open_trade_value) - 1
return float(f"{profit_ratio:.8f}")
@ -572,23 +569,6 @@ class LocalTrade():
else:
return None
@staticmethod
def get_trades(trade_filter=None) -> Query:
"""
Helper function to query Trades using filters.
:param trade_filter: Optional filter to apply to trades
Can be either a Filter object, or a List of filters
e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
e.g. `(trade_filter=Trade.id == trade_id)`
:return: unsorted query object
"""
if trade_filter is not None:
if not isinstance(trade_filter, list):
trade_filter = [trade_filter]
return Trade.query.filter(*trade_filter)
else:
return Trade.query
@staticmethod
def get_trades_proxy(*, pair: str = None, is_open: bool = None,
open_date: datetime = None, close_date: datetime = None,
@ -611,7 +591,7 @@ class LocalTrade():
else:
# Not used during backtesting, but might be used by a strategy
sel_trades = [trade for trade in LocalTrade.trades + LocalTrade.trades_open]
sel_trades = list(LocalTrade.trades + LocalTrade.trades_open)
if pair:
sel_trades = [trade for trade in sel_trades if trade.pair == pair]
@ -641,83 +621,7 @@ class LocalTrade():
"""
Query trades from persistence layer
"""
return Trade.get_trades(Trade.is_open.is_(True)).all()
@staticmethod
def get_open_order_trades():
"""
Returns all open trades
"""
return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
@staticmethod
def get_open_trades_without_assigned_fees():
"""
Returns all open trades which don't have open fees set correctly
"""
return Trade.get_trades([Trade.fee_open_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(True),
]).all()
@staticmethod
def get_sold_trades_without_assigned_fees():
"""
Returns all closed trades which don't have fees set correctly
"""
return Trade.get_trades([Trade.fee_close_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(False),
]).all()
@staticmethod
def total_open_trades_stakes() -> float:
"""
Calculates total invested amount in open trades
in stake currency
"""
if Trade.use_db:
total_open_stake_amount = Trade.session.query(
func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
else:
total_open_stake_amount = sum(
t.stake_amount for t in Trade.get_trades_proxy(is_open=True))
return total_open_stake_amount or 0
@staticmethod
def get_overall_performance() -> List[Dict[str, Any]]:
"""
Returns List of dicts containing all Trades, including profit and trade count
"""
pair_rates = Trade.session.query(
Trade.pair,
func.sum(Trade.close_profit).label('profit_sum'),
func.count(Trade.pair).label('count')
).filter(Trade.is_open.is_(False))\
.group_by(Trade.pair) \
.order_by(desc('profit_sum')) \
.all()
return [
{
'pair': pair,
'profit': rate,
'count': count
}
for pair, rate, count in pair_rates
]
@staticmethod
def get_best_pair():
"""
Get best pair with closed trade.
:returns: Tuple containing (pair, profit_sum)
"""
best_pair = Trade.session.query(
Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
).filter(Trade.is_open.is_(False)) \
.group_by(Trade.pair) \
.order_by(desc('profit_sum')).first()
return best_pair
return Trade.get_trades_proxy(is_open=True)
@staticmethod
def stoploss_reinitialization(desired_stoploss):
@ -754,15 +658,15 @@ class Trade(_DECL_BASE, LocalTrade):
orders = relationship("Order", order_by="Order.id", cascade="all, delete-orphan")
exchange = Column(String, nullable=False)
pair = Column(String, nullable=False, index=True)
exchange = Column(String(25), nullable=False)
pair = Column(String(25), nullable=False, index=True)
is_open = Column(Boolean, nullable=False, default=True, index=True)
fee_open = Column(Float, nullable=False, default=0.0)
fee_open_cost = Column(Float, nullable=True)
fee_open_currency = Column(String, nullable=True)
fee_open_currency = Column(String(25), nullable=True)
fee_close = Column(Float, nullable=False, default=0.0)
fee_close_cost = Column(Float, nullable=True)
fee_close_currency = Column(String, nullable=True)
fee_close_currency = Column(String(25), nullable=True)
open_rate = Column(Float)
open_rate_requested = Column(Float)
# open_trade_value - calculated via _calc_open_trade_value
@ -776,7 +680,7 @@ class Trade(_DECL_BASE, LocalTrade):
amount_requested = Column(Float)
open_date = Column(DateTime, nullable=False, default=datetime.utcnow)
close_date = Column(DateTime)
open_order_id = Column(String)
open_order_id = Column(String(255))
# absolute value of the stop loss
stop_loss = Column(Float, nullable=True, default=0.0)
# percentage value of the stop loss
@ -786,16 +690,16 @@ class Trade(_DECL_BASE, LocalTrade):
# percentage value of the initial stop loss
initial_stop_loss_pct = Column(Float, nullable=True)
# stoploss order id which is on exchange
stoploss_order_id = Column(String, nullable=True, index=True)
stoploss_order_id = Column(String(255), nullable=True, index=True)
# last update time of the stoploss order on exchange
stoploss_last_update = Column(DateTime, nullable=True)
# absolute value of the highest reached price
max_rate = Column(Float, nullable=True, default=0.0)
# Lowest price reached
min_rate = Column(Float, nullable=True)
sell_reason = Column(String, nullable=True)
sell_order_status = Column(String, nullable=True)
strategy = Column(String, nullable=True)
sell_reason = Column(String(100), nullable=True)
sell_order_status = Column(String(100), nullable=True)
strategy = Column(String(100), nullable=True)
timeframe = Column(Integer, nullable=True)
def __init__(self, **kwargs):
@ -805,17 +709,17 @@ class Trade(_DECL_BASE, LocalTrade):
def delete(self) -> None:
for order in self.orders:
Order.session.delete(order)
Order.query.session.delete(order)
Trade.session.delete(self)
Trade.session.flush()
Trade.query.session.delete(self)
Trade.query.session.flush()
@staticmethod
def get_trades_proxy(*, pair: str = None, is_open: bool = None,
open_date: datetime = None, close_date: datetime = None,
) -> List['LocalTrade']:
"""
Helper function to query Trades.
Helper function to query Trades.j
Returns a List of trades, filtered on the parameters given.
In live mode, converts the filter to a database query and returns all rows
In Backtest mode, uses filters on Trade.trades to get the result.
@ -840,6 +744,109 @@ class Trade(_DECL_BASE, LocalTrade):
close_date=close_date
)
@staticmethod
def get_trades(trade_filter=None) -> Query:
"""
Helper function to query Trades using filters.
NOTE: Not supported in Backtesting.
:param trade_filter: Optional filter to apply to trades
Can be either a Filter object, or a List of filters
e.g. `(trade_filter=[Trade.id == trade_id, Trade.is_open.is_(True),])`
e.g. `(trade_filter=Trade.id == trade_id)`
:return: unsorted query object
"""
if not Trade.use_db:
raise NotImplementedError('`Trade.get_trades()` not supported in backtesting mode.')
if trade_filter is not None:
if not isinstance(trade_filter, list):
trade_filter = [trade_filter]
return Trade.query.filter(*trade_filter)
else:
return Trade.query
@staticmethod
def get_open_order_trades():
"""
Returns all open trades
NOTE: Not supported in Backtesting.
"""
return Trade.get_trades(Trade.open_order_id.isnot(None)).all()
@staticmethod
def get_open_trades_without_assigned_fees():
"""
Returns all open trades which don't have open fees set correctly
NOTE: Not supported in Backtesting.
"""
return Trade.get_trades([Trade.fee_open_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(True),
]).all()
@staticmethod
def get_sold_trades_without_assigned_fees():
"""
Returns all closed trades which don't have fees set correctly
NOTE: Not supported in Backtesting.
"""
return Trade.get_trades([Trade.fee_close_currency.is_(None),
Trade.orders.any(),
Trade.is_open.is_(False),
]).all()
@staticmethod
def total_open_trades_stakes() -> float:
"""
Calculates total invested amount in open trades
in stake currency
"""
if Trade.use_db:
total_open_stake_amount = Trade.query.with_entities(
func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
else:
total_open_stake_amount = sum(
t.stake_amount for t in LocalTrade.get_trades_proxy(is_open=True))
return total_open_stake_amount or 0
@staticmethod
def get_overall_performance() -> List[Dict[str, Any]]:
"""
Returns List of dicts containing all Trades, including profit and trade count
NOTE: Not supported in Backtesting.
"""
pair_rates = Trade.query.with_entities(
Trade.pair,
func.sum(Trade.close_profit).label('profit_sum'),
func.sum(Trade.close_profit_abs).label('profit_sum_abs'),
func.count(Trade.pair).label('count')
).filter(Trade.is_open.is_(False))\
.group_by(Trade.pair) \
.order_by(desc('profit_sum_abs')) \
.all()
return [
{
'pair': pair,
'profit': profit,
'profit_abs': profit_abs,
'count': count
}
for pair, profit, profit_abs, count in pair_rates
]
@staticmethod
def get_best_pair():
"""
Get best pair with closed trade.
NOTE: Not supported in Backtesting.
:returns: Tuple containing (pair, profit_sum)
"""
best_pair = Trade.query.with_entities(
Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
).filter(Trade.is_open.is_(False)) \
.group_by(Trade.pair) \
.order_by(desc('profit_sum')).first()
return best_pair
class PairLock(_DECL_BASE):
"""
@ -849,8 +856,8 @@ class PairLock(_DECL_BASE):
id = Column(Integer, primary_key=True)
pair = Column(String, nullable=False, index=True)
reason = Column(String, nullable=True)
pair = Column(String(25), nullable=False, index=True)
reason = Column(String(255), nullable=True)
# Time the pair was locked (start time)
lock_time = Column(DateTime, nullable=False)
# Time until the pair is locked (end time)

6
freqtrade/persistence/pairlock_middleware.py
View File

@ -48,8 +48,8 @@ class PairLocks():
active=True
)
if PairLocks.use_db:
PairLock.session.add(lock)
PairLock.session.flush()
PairLock.query.session.add(lock)
PairLock.query.session.flush()
else:
PairLocks.locks.append(lock)
@ -99,7 +99,7 @@ class PairLocks():
for lock in locks:
lock.active = False
if PairLocks.use_db:
PairLock.session.flush()
PairLock.query.session.flush()
@staticmethod
def is_global_lock(now: Optional[datetime] = None) -> bool:

19
freqtrade/plot/plotting.py
View File

@ -77,7 +77,8 @@ def init_plotscript(config, markets: List, startup_candles: int = 0):
)
except ValueError as e:
raise OperationalException(e) from e
trades = trim_dataframe(trades, timerange, 'open_date')
if not trades.empty:
trades = trim_dataframe(trades, timerange, 'open_date')
return {"ohlcv": data,
"trades": trades,
@ -441,7 +442,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
trades: pd.DataFrame, timeframe: str) -> go.Figure:
trades: pd.DataFrame, timeframe: str, stake_currency: str) -> go.Figure:
# Combine close-values for all pairs, rename columns to "pair"
df_comb = combine_dataframes_with_mean(data, "close")
@ -466,8 +467,8 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
subplot_titles=["AVG Close Price", "Combined Profit", "Profit per pair"])
fig['layout'].update(title="Freqtrade Profit plot")
fig['layout']['yaxis1'].update(title='Price')
fig['layout']['yaxis2'].update(title='Profit')
fig['layout']['yaxis3'].update(title='Profit')
fig['layout']['yaxis2'].update(title=f'Profit {stake_currency}')
fig['layout']['yaxis3'].update(title=f'Profit {stake_currency}')
fig['layout']['xaxis']['rangeslider'].update(visible=False)
fig.add_trace(avgclose, 1, 1)
@ -540,8 +541,11 @@ def load_and_plot_trades(config: Dict[str, Any]):
df_analyzed = strategy.analyze_ticker(data, {'pair': pair})
df_analyzed = trim_dataframe(df_analyzed, timerange)
trades_pair = trades.loc[trades['pair'] == pair]
trades_pair = extract_trades_of_period(df_analyzed, trades_pair)
if not trades.empty:
trades_pair = trades.loc[trades['pair'] == pair]
trades_pair = extract_trades_of_period(df_analyzed, trades_pair)
else:
trades_pair = trades
fig = generate_candlestick_graph(
pair=pair,
@ -581,6 +585,7 @@ def plot_profit(config: Dict[str, Any]) -> None:
# Create an average close price of all the pairs that were involved.
# this could be useful to gauge the overall market trend
fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'],
trades, config.get('timeframe', '5m'))
trades, config.get('timeframe', '5m'),
config.get('stake_currency', ''))
store_plot_file(fig, filename='freqtrade-profit-plot.html',
directory=config['user_data_dir'] / 'plot', auto_open=True)

6
freqtrade/plugins/pairlist/AgeFilter.py
View File

@ -71,14 +71,14 @@ class AgeFilter(IPairList):
daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
if not self._validate_pair_loc(p, daily_candles):
pairlist.remove(p)
logger.info(f"Validated {len(pairlist)} pairs.")
self.log_once(f"Validated {len(pairlist)} pairs.", logger.info)
return pairlist
def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
"""
Validate age for the ticker
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.load_markets()
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:return: True if the pair can stay, false if it should be removed
"""
# Check symbol in cache
@ -86,7 +86,7 @@ class AgeFilter(IPairList):
return True
if daily_candles is not None:
if len(daily_candles) > self._min_days_listed:
if len(daily_candles) >= self._min_days_listed:
# We have fetched at least the minimum required number of daily candles
# Add to cache, store the time we last checked this symbol
self._symbolsChecked[pair] = int(arrow.utcnow().float_timestamp) * 1000

13
freqtrade/plugins/pairlist/IPairList.py
View File

@ -7,7 +7,7 @@ from copy import deepcopy
from typing import Any, Dict, List
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import market_is_active
from freqtrade.exchange import Exchange, market_is_active
from freqtrade.mixins import LoggingMixin
@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
class IPairList(LoggingMixin, ABC):
def __init__(self, exchange, pairlistmanager,
def __init__(self, exchange: Exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
"""
@ -28,7 +28,7 @@ class IPairList(LoggingMixin, ABC):
"""
self._enabled = True
self._exchange = exchange
self._exchange: Exchange = exchange
self._pairlistmanager = pairlistmanager
self._config = config
self._pairlistconfig = pairlistconfig
@ -68,12 +68,12 @@ class IPairList(LoggingMixin, ABC):
filter_pairlist() method.
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.load_markets()
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:return: True if the pair can stay, false if it should be removed
"""
raise NotImplementedError()
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
def gen_pairlist(self, tickers: Dict) -> List[str]:
"""
Generate the pairlist.
@ -84,8 +84,7 @@ class IPairList(LoggingMixin, ABC):
it will raise the exception if a Pairlist Handler is used at the first
position in the chain.
:param cached_pairlist: Previously generated pairlist (cached)
:param tickers: Tickers (from exchange.get_tickers()).
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs
"""
raise OperationalException("This Pairlist Handler should not be used "

14
freqtrade/plugins/pairlist/PerformanceFilter.py
View File

@ -2,7 +2,7 @@
Performance pair list filter
"""
import logging
from typing import Any, Dict, List
from typing import Dict, List
import pandas as pd
@ -15,11 +15,6 @@ logger = logging.getLogger(__name__)
class PerformanceFilter(IPairList):
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
@property
def needstickers(self) -> bool:
"""
@ -44,7 +39,12 @@ class PerformanceFilter(IPairList):
:return: new allowlist
"""
# Get the trading performance for pairs from database
performance = pd.DataFrame(Trade.get_overall_performance())
try:
performance = pd.DataFrame(Trade.get_overall_performance())
except AttributeError:
# Performancefilter does not work in backtesting.
self.log_once("PerformanceFilter is not available in this mode.", logger.warning)
return pairlist
# Skip performance-based sorting if no performance data is available
if len(performance) == 0:

2
freqtrade/plugins/pairlist/PrecisionFilter.py
View File

@ -48,7 +48,7 @@ class PrecisionFilter(IPairList):
Check if pair has enough room to add a stoploss to avoid "unsellable" buys of very
low value pairs.
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.load_markets()
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:return: True if the pair can stay, false if it should be removed
"""
stop_price = ticker['ask'] * self._stoploss

38
freqtrade/plugins/pairlist/PriceFilter.py
View File

@ -27,9 +27,13 @@ class PriceFilter(IPairList):
self._max_price = pairlistconfig.get('max_price', 0)
if self._max_price < 0:
raise OperationalException("PriceFilter requires max_price to be >= 0")
self._max_value = pairlistconfig.get('max_value', 0)
if self._max_value < 0:
raise OperationalException("PriceFilter requires max_value to be >= 0")
self._enabled = ((self._low_price_ratio > 0) or
(self._min_price > 0) or
(self._max_price > 0))
(self._max_price > 0) or
(self._max_value > 0))
@property
def needstickers(self) -> bool:
@ -51,6 +55,8 @@ class PriceFilter(IPairList):
active_price_filters.append(f"below {self._min_price:.8f}")
if self._max_price != 0:
active_price_filters.append(f"above {self._max_price:.8f}")
if self._max_value != 0:
active_price_filters.append(f"Value above {self._max_value:.8f}")
if len(active_price_filters):
return f"{self.name} - Filtering pairs priced {' or '.join(active_price_filters)}."
@ -61,7 +67,7 @@ class PriceFilter(IPairList):
"""
Check if if one price-step (pip) is > than a certain barrier.
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.load_markets()
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:return: True if the pair can stay, false if it should be removed
"""
if ticker.get('last', None) is None or ticker.get('last') == 0:
@ -79,6 +85,32 @@ class PriceFilter(IPairList):
f"because 1 unit is {changeperc * 100:.3f}%", logger.info)
return False
# Perform low_amount check
if self._max_value != 0:
price = ticker['last']
market = self._exchange.markets[pair]
limits = market['limits']
if ('amount' in limits and 'min' in limits['amount']
and limits['amount']['min'] is not None):
min_amount = limits['amount']['min']
min_precision = market['precision']['amount']
min_value = min_amount * price
if self._exchange.precisionMode == 4:
# tick size
next_value = (min_amount + min_precision) * price
else:
# Decimal places
min_precision = pow(0.1, min_precision)
next_value = (min_amount + min_precision) * price
diff = next_value - min_value
if diff > self._max_value:
self.log_once(f"Removed {pair} from whitelist, "
f"because min value change of {diff} > {self._max_value}.",
logger.info)
return False
# Perform min_price check.
if self._min_price != 0:
if ticker['last'] < self._min_price:
@ -89,7 +121,7 @@ class PriceFilter(IPairList):
# Perform max_price check.
if self._max_price != 0:
if ticker['last'] > self._max_price:
self.log_once(f"Removed {ticker['symbol']} from whitelist, "
self.log_once(f"Removed {pair} from whitelist, "
f"because last price > {self._max_price:.8f}", logger.info)
return False

2
freqtrade/plugins/pairlist/SpreadFilter.py
View File

@ -40,7 +40,7 @@ class SpreadFilter(IPairList):
"""
Validate spread for the ticker
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.load_markets()
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:return: True if the pair can stay, false if it should be removed
"""
if 'bid' in ticker and 'ask' in ticker and ticker['ask']:

5
freqtrade/plugins/pairlist/StaticPairList.py
View File

@ -42,11 +42,10 @@ class StaticPairList(IPairList):
"""
return f"{self.name}"
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
def gen_pairlist(self, tickers: Dict) -> List[str]:
"""
Generate the pairlist
:param cached_pairlist: Previously generated pairlist (cached)
:param tickers: Tickers (from exchange.get_tickers()).
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs
"""
if self._allow_inactive:

121
freqtrade/plugins/pairlist/VolatilityFilter.py Normal file
View File

@ -0,0 +1,121 @@
"""
Volatility pairlist filter
"""
import logging
import sys
from copy import deepcopy
from typing import Any, Dict, List, Optional
import arrow
import numpy as np
from cachetools.ttl import TTLCache
from pandas import DataFrame
from freqtrade.exceptions import OperationalException
from freqtrade.misc import plural
from freqtrade.plugins.pairlist.IPairList import IPairList
logger = logging.getLogger(__name__)
class VolatilityFilter(IPairList):
'''
Filters pairs by volatility
'''
def __init__(self, exchange, pairlistmanager,
config: Dict[str, Any], pairlistconfig: Dict[str, Any],
pairlist_pos: int) -> None:
super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
self._days = pairlistconfig.get('lookback_days', 10)
self._min_volatility = pairlistconfig.get('min_volatility', 0)
self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize)
self._refresh_period = pairlistconfig.get('refresh_period', 1440)
self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
if self._days < 1:
raise OperationalException("VolatilityFilter requires lookback_days to be >= 1")
if self._days > exchange.ohlcv_candle_limit('1d'):
raise OperationalException("VolatilityFilter requires lookback_days to not "
"exceed exchange max request size "
f"({exchange.ohlcv_candle_limit('1d')})")
@property
def needstickers(self) -> bool:
"""
Boolean property defining if tickers are necessary.
If no Pairlist requires tickers, an empty List is passed
as tickers argument to filter_pairlist
"""
return False
def short_desc(self) -> str:
"""
Short whitelist method description - used for startup-messages
"""
return (f"{self.name} - Filtering pairs with volatility range "
f"{self._min_volatility}-{self._max_volatility} "
f" the last {self._days} {plural(self._days, 'day')}.")
def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
"""
Validate trading range
:param pairlist: pairlist to filter or sort
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: new allowlist
"""
needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
since_ms = int(arrow.utcnow()
.floor('day')
.shift(days=-self._days - 1)
.float_timestamp) * 1000
# Get all candles
candles = {}
if needed_pairs:
candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
cache=False)
if self._enabled:
for p in deepcopy(pairlist):
daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
if not self._validate_pair_loc(p, daily_candles):
pairlist.remove(p)
return pairlist
def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
"""
Validate trading range
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:return: True if the pair can stay, false if it should be removed
"""
# Check symbol in cache
cached_res = self._pair_cache.get(pair, None)
if cached_res is not None:
return cached_res
result = False
if daily_candles is not None and not daily_candles.empty:
returns = (np.log(daily_candles.close / daily_candles.close.shift(-1)))
returns.fillna(0, inplace=True)
volatility_series = returns.rolling(window=self._days).std()*np.sqrt(self._days)
volatility_avg = volatility_series.mean()
if self._min_volatility <= volatility_avg <= self._max_volatility:
result = True
else:
self.log_once(f"Removed {pair} from whitelist, because volatility "
f"over {self._days} {plural(self._days, 'day')} "
f"is: {volatility_avg:.3f} "
f"which is not in the configured range of "
f"{self._min_volatility}-{self._max_volatility}.",
logger.info)
result = False
self._pair_cache[pair] = result
return result

24
freqtrade/plugins/pairlist/VolumePairList.py
View File

@ -4,9 +4,10 @@ Volume PairList provider
Provides dynamic pair list based on trade volumes
"""
import logging
from datetime import datetime
from typing import Any, Dict, List
from cachetools.ttl import TTLCache
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.IPairList import IPairList
@ -33,7 +34,8 @@ class VolumePairList(IPairList):
self._number_pairs = self._pairlistconfig['number_assets']
self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
self._min_value = self._pairlistconfig.get('min_value', 0)
self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
if not self._exchange.exchange_has('fetchTickers'):
raise OperationalException(
@ -63,17 +65,19 @@ class VolumePairList(IPairList):
"""
return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
def gen_pairlist(self, tickers: Dict) -> List[str]:
"""
Generate the pairlist
:param cached_pairlist: Previously generated pairlist (cached)
:param tickers: Tickers (from exchange.get_tickers()).
:param tickers: Tickers (from exchange.get_tickers()). May be cached.
:return: List of pairs
"""
# Generate dynamic whitelist
# Must always run if this pairlist is not the first in the list.
if self._last_refresh + self.refresh_period < datetime.now().timestamp():
self._last_refresh = int(datetime.now().timestamp())
pairlist = self._pair_cache.get('pairlist')
if pairlist:
# Item found - no refresh necessary
return pairlist
else:
# Use fresh pairlist
# Check if pair quote currency equals to the stake currency.
@ -82,9 +86,9 @@ class VolumePairList(IPairList):
if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
and v[self._sort_key] is not None)]
pairlist = [s['symbol'] for s in filtered_tickers]
else:
# Use the cached pairlist if it's not time yet to refresh
pairlist = cached_pairlist
pairlist = self.filter_pairlist(pairlist, tickers)
self._pair_cache['pairlist'] = pairlist
return pairlist

7
freqtrade/plugins/pairlist/rangestabilityfilter.py
View File

@ -83,12 +83,13 @@ class RangeStabilityFilter(IPairList):
"""
Validate trading range
:param pair: Pair that's currently validated
:param ticker: ticker dict as returned from ccxt.load_markets()
:param ticker: ticker dict as returned from ccxt.fetch_tickers()
:return: True if the pair can stay, false if it should be removed
"""
# Check symbol in cache
if pair in self._pair_cache:
return self._pair_cache[pair]
cached_res = self._pair_cache.get(pair, None)
if cached_res is not None:
return cached_res
result = False
if daily_candles is not None and not daily_candles.empty:

20
freqtrade/plugins/pairlistmanager.py
View File

@ -3,7 +3,7 @@ PairList manager class
"""
import logging
from copy import deepcopy
from typing import Any, Dict, List
from typing import Dict, List
from cachetools import TTLCache, cached
@ -79,11 +79,8 @@ class PairListManager():
if self._tickers_needed:
tickers = self._get_cached_tickers()
# Adjust whitelist if filters are using tickers
pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
# Generate the pairlist with first Pairlist Handler in the chain
pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers)
pairlist = self._pairlist_handlers[0].gen_pairlist(tickers)
# Process all Pairlist Handlers in the chain
for pairlist_handler in self._pairlist_handlers:
@ -95,19 +92,6 @@ class PairListManager():
self._whitelist = pairlist
def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]:
"""
Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
pairs that do not have ticker available
"""
if self._tickers_needed:
# Copy list since we're modifying this list
for p in deepcopy(pairlist):
if p not in tickers:
pairlist.remove(p)
return pairlist
def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
"""
Verify and remove items from pairlist - returning a filtered pairlist.

4
freqtrade/plugins/protections/cooldown_period.py
View File

@ -1,7 +1,6 @@
import logging
from datetime import datetime, timedelta
from typing import Any, Dict
from freqtrade.persistence import Trade
from freqtrade.plugins.protections import IProtection, ProtectionReturn
@ -15,9 +14,6 @@ class CooldownPeriod(IProtection):
has_global_stop: bool = False
has_local_stop: bool = True
def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None:
super().__init__(config, protection_config)
def _reason(self) -> str:
"""
LockReason to use

2
freqtrade/plugins/protections/max_drawdown_protection.py
View File

@ -61,7 +61,7 @@ class MaxDrawdown(IProtection):
if drawdown > self._max_allowed_drawdown:
self.log_once(
f"Trading stopped due to Max Drawdown {drawdown:.2f} < {self._max_allowed_drawdown}"
f"Trading stopped due to Max Drawdown {drawdown:.2f} > {self._max_allowed_drawdown}"
f" within {self.lookback_period_str}.", logger.info)
until = self.calculate_lock_end(trades, self._stop_duration)

2
freqtrade/resolvers/iresolver.py
View File

@ -61,7 +61,7 @@ class IResolver:
module = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(module) # type: ignore # importlib does not use typehints
except (ModuleNotFoundError, SyntaxError, ImportError) as err:
except (ModuleNotFoundError, SyntaxError, ImportError, NameError) as err:
# Catch errors in case a specific module is not installed
logger.warning(f"Could not import {module_path} due to '{err}'")
if enum_failed:

6
freqtrade/resolvers/strategy_resolver.py
View File

@ -196,9 +196,9 @@ class StrategyResolver(IResolver):
strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
if any([x == 2 for x in [strategy._populate_fun_len,
strategy._buy_fun_len,
strategy._sell_fun_len]]):
if any(x == 2 for x in [strategy._populate_fun_len,
strategy._buy_fun_len,
strategy._sell_fun_len]):
strategy.INTERFACE_VERSION = 1
return strategy

Some files were not shown because too many files have changed in this diff Show More