Be explicit with space assignment in documentation

closes #4852

.dockerignore

@@ -1,9 +1,8 @@
 .git
 .gitignore
 Dockerfile
+Dockerfile.armhf
 .dockerignore
-config.json*
-*.sqlite
 .coveragerc
 .eggs
 .github
@@ -13,4 +12,13 @@ CONTRIBUTING.md
 MANIFEST.in
 README.md
 freqtrade.service
+freqtrade.egg-info
+
+config.json*
+*.sqlite
 user_data
+*.log
+
+.vscode
+.mypy_cache
+.ipynb_checkpoints

.gitattributes

@@ -0,0 +1,3 @@
+*.py eol=lf
+*.sh eol=lf
+*.ps1 eol=crlf

.github/workflows/ci.yml

@@ -148,6 +148,7 @@ jobs:
 
     - name: Installation - macOS
       run: |
+        brew update
         brew install hdf5 c-blosc
         python -m pip install --upgrade pip
         export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH
@@ -300,7 +301,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Cleanup previous runs on this branch
-      uses: rokroskar/workflow-run-cleanup-action@v0.2.2
+      uses: rokroskar/workflow-run-cleanup-action@v0.3.2
       if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/stable' && github.repository == 'freqtrade/freqtrade'"
       env:
         GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
@@ -310,9 +311,18 @@ jobs:
     needs: [ build_linux, build_macos, build_windows, docs_check ]
     runs-on: ubuntu-20.04
     steps:
+
+    - name: Check user permission
+      id: check
+      uses: scherermichael-oss/action-has-permission@1.0.6
+      with:
+        required-permission: write
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
     - name: Slack Notification
       uses: lazy-actions/slatify@v3.0.0
-      if: always() && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
+      if: always() && steps.check.outputs.has-permission && ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)
       with:
         type: ${{ job.status }}
         job_name: '*Freqtrade CI*'

Dockerfile

@@ -1,14 +1,23 @@
-FROM python:3.9.2-slim-buster as base
+FROM python:3.9.4-slim-buster as base
 
 # Setup env
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONFAULTHANDLER 1
-ENV PATH=/root/.local/bin:$PATH
+ENV PATH=/home/ftuser/.local/bin:$PATH
+ENV FT_APP_ENV="docker"
 
 # Prepare environment
-RUN mkdir /freqtrade
+RUN mkdir /freqtrade \
+  && apt update \
+  && apt install -y sudo \
+  && apt-get clean \
+  && useradd -u 1000 -G sudo -U -m ftuser \
+  && chown ftuser:ftuser /freqtrade \
+  # Allow sudoers
+  && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
+
 WORKDIR /freqtrade
 
 # Install dependencies
@@ -24,7 +33,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Install dependencies
-COPY requirements.txt requirements-hyperopt.txt /freqtrade/
+COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/
+USER ftuser
 RUN  pip install --user --no-cache-dir numpy \
   && pip install --user --no-cache-dir -r requirements-hyperopt.txt
 
@@ -33,13 +43,13 @@ FROM base as runtime-image
 COPY --from=python-deps /usr/local/lib /usr/local/lib
 ENV LD_LIBRARY_PATH /usr/local/lib
 
-COPY --from=python-deps /root/.local /root/.local
-
-
+COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local
 
+USER ftuser
 # Install and execute
-COPY . /freqtrade/
-RUN pip install -e . --no-cache-dir \
+COPY --chown=ftuser:ftuser . /freqtrade/
+
+RUN pip install -e . --user --no-cache-dir \
   && mkdir /freqtrade/user_data/ \
   && freqtrade install-ui
 

Dockerfile.armhf

@@ -1,19 +1,24 @@
-FROM --platform=linux/arm/v7 python:3.7.9-slim-buster as base
+FROM --platform=linux/arm/v7 python:3.7.10-slim-buster as base
 
 # Setup env
 ENV LANG C.UTF-8
 ENV LC_ALL C.UTF-8
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONFAULTHANDLER 1
-ENV PATH=/root/.local/bin:$PATH
+ENV PATH=/home/ftuser/.local/bin:$PATH
+ENV FT_APP_ENV="docker"
 
 # Prepare environment
-RUN mkdir /freqtrade
-WORKDIR /freqtrade
+RUN mkdir /freqtrade \
+  && apt-get update \
+  && apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \
+  && apt-get clean \
+  && useradd -u 1000 -G sudo -U -m ftuser \
+  && chown ftuser:ftuser /freqtrade \
+  # Allow sudoers
+  && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
 
-RUN  apt-get update \
-  && apt-get -y install libatlas3-base curl sqlite3 \
-  && apt-get clean
+WORKDIR /freqtrade
 
 # Install dependencies
 FROM base as python-deps
@@ -28,7 +33,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib*
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Install dependencies
-COPY requirements.txt /freqtrade/
+COPY --chown=ftuser:ftuser requirements.txt /freqtrade/
+USER ftuser
 RUN  pip install --user --no-cache-dir numpy \
   && pip install --user --no-cache-dir -r requirements.txt
 
@@ -37,13 +43,14 @@ FROM base as runtime-image
 COPY --from=python-deps /usr/local/lib /usr/local/lib
 ENV LD_LIBRARY_PATH /usr/local/lib
 
-COPY --from=python-deps /root/.local /root/.local
+COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local
 
+USER ftuser
 # Install and execute
-COPY . /freqtrade/
-RUN apt-get install -y libhdf5-serial-dev \
-  && apt-get clean \
-  && pip install -e . --no-cache-dir \
+COPY --chown=ftuser:ftuser . /freqtrade/
+
+RUN pip install -e . --user --no-cache-dir \
+  && mkdir /freqtrade/user_data/ \
   && freqtrade install-ui
 
 ENTRYPOINT ["freqtrade"]

README.md

@@ -1,4 +1,4 @@
-# Freqtrade
+# ![freqtrade](docs/assets/freqtrade_poweredby.svg)
 
 [![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/)
 [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop)

config_ftx.json.example

@@ -0,0 +1,99 @@
+{
+    "max_open_trades": 3,
+    "stake_currency": "USD",
+    "stake_amount": 50,
+    "tradable_balance_ratio": 0.99,
+    "fiat_display_currency": "USD",
+    "timeframe": "5m",
+    "dry_run": true,
+    "cancel_open_orders_on_exit": false,
+    "unfilledtimeout": {
+        "buy": 10,
+        "sell": 30
+    },
+    "bid_strategy": {
+        "ask_last_balance": 0.0,
+        "use_order_book": false,
+        "order_book_top": 1,
+        "check_depth_of_market": {
+            "enabled": false,
+            "bids_to_ask_delta": 1
+        }
+    },
+    "ask_strategy": {
+        "use_order_book": false,
+        "order_book_min": 1,
+        "order_book_max": 1,
+        "use_sell_signal": true,
+        "sell_profit_only": false,
+        "ignore_roi_if_buy_signal": false
+    },
+    "exchange": {
+        "name": "ftx",
+        "key": "your_exchange_key",
+        "secret": "your_exchange_secret",
+        "ccxt_config": {"enableRateLimit": true},
+        "ccxt_async_config": {
+            "enableRateLimit": true,
+            "rateLimit": 50
+        },
+        "pair_whitelist": [
+            "BTC/USD",
+            "ETH/USD",
+            "BNB/USD",
+            "USDT/USD",
+            "LTC/USD",
+            "SRM/USD",
+            "SXP/USD",
+            "XRP/USD",
+            "DOGE/USD",
+            "1INCH/USD",
+            "CHZ/USD",
+            "MATIC/USD",
+            "LINK/USD",
+            "OXY/USD",
+            "SUSHI/USD"
+        ],
+        "pair_blacklist": [
+            "FTT/USD"
+        ]
+    },
+    "pairlists": [
+        {"method": "StaticPairList"}
+    ],
+    "edge": {
+        "enabled": false,
+        "process_throttle_secs": 3600,
+        "calculate_since_number_of_days": 7,
+        "allowed_risk": 0.01,
+        "stoploss_range_min": -0.01,
+        "stoploss_range_max": -0.1,
+        "stoploss_range_step": -0.01,
+        "minimum_winrate": 0.60,
+        "minimum_expectancy": 0.20,
+        "min_trade_number": 10,
+        "max_trade_duration_minute": 1440,
+        "remove_pumps": false
+    },
+    "telegram": {
+        "enabled": false,
+        "token": "your_telegram_token",
+        "chat_id": "your_telegram_chat_id"
+    },
+    "api_server": {
+        "enabled": false,
+        "listen_ip_address": "127.0.0.1",
+        "listen_port": 8080,
+        "verbosity": "error",
+        "jwt_secret_key": "somethingrandom",
+        "CORS_origins": [],
+        "username": "freqtrader",
+        "password": "SuperSecurePassword"
+    },
+    "bot_name": "freqtrade",
+    "initial_state": "running",
+    "forcebuy_enable": false,
+    "internals": {
+        "process_throttle_secs": 5
+    }
+}

config_full.json.example

@@ -163,7 +163,9 @@
             "warning": "on",
             "startup": "on",
             "buy": "on",
+            "buy_fill": "on",
             "sell": "on",
+            "sell_fill": "on",
             "buy_cancel": "on",
             "sell_cancel": "on"
         }

docker-compose.yml

@@ -9,7 +9,7 @@ services:
     # Build step - only needed when additional dependencies are needed
     # build:
     #   context: .
-    #   dockerfile: "./docker/Dockerfile.technical"
+    #   dockerfile: "./docker/Dockerfile.custom"
     restart: unless-stopped
     container_name: freqtrade
     volumes:

docker/Dockerfile.custom

@@ -0,0 +1,10 @@
+FROM freqtradeorg/freqtrade:develop
+
+# Switch user to root if you must install something from apt
+# Don't forget to switch the user back below!
+# USER root
+
+# The below dependency - pyti - serves as an example. Please use whatever you need!
+RUN pip install --user pyti
+
+# USER ftuser

docker/Dockerfile.develop

@@ -3,8 +3,8 @@ FROM freqtradeorg/freqtrade:develop
 # Install dependencies
 COPY requirements-dev.txt /freqtrade/
 
-RUN pip install numpy --no-cache-dir \
-  && pip install -r requirements-dev.txt --no-cache-dir
+RUN pip install numpy --user --no-cache-dir \
+  && pip install -r requirements-dev.txt --user --no-cache-dir
 
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

docker/Dockerfile.jupyter

@@ -1,7 +1,7 @@
 FROM freqtradeorg/freqtrade:develop_plot
 
 
-RUN pip install jupyterlab --no-cache-dir
+RUN pip install jupyterlab --user --no-cache-dir
 
 # Empty the ENTRYPOINT to allow all commands
 ENTRYPOINT []

docker/Dockerfile.plot

@@ -4,4 +4,4 @@ FROM freqtradeorg/freqtrade:${sourceimage}
 # Install dependencies
 COPY requirements-plot.txt /freqtrade/
 
-RUN pip install -r requirements-plot.txt --no-cache-dir
+RUN pip install -r requirements-plot.txt --user --no-cache-dir

docker/Dockerfile.technical

@@ -1,6 +0,0 @@
-FROM freqtradeorg/freqtrade:develop
-
-RUN apt-get update \
-    && apt-get -y install git \
-    && apt-get clean \
-    && pip install git+https://github.com/freqtrade/technical

docs/advanced-hyperopt.md

@@ -4,79 +4,6 @@ This page explains some advanced Hyperopt topics that may require higher
 coding skills and Python knowledge than creation of an ordinal hyperoptimization
 class.
 
-## Derived hyperopt classes
-
-Custom hyperopt classes can be derived in the same way [it can be done for strategies](strategy-customization.md#derived-strategies).
-
-Applying to hyperoptimization, as an example, you may override how dimensions are defined in your optimization hyperspace:
-
-```python
-class MyAwesomeHyperOpt(IHyperOpt):
-    ...
-    # Uses default stoploss dimension
-
-class MyAwesomeHyperOpt2(MyAwesomeHyperOpt):
-    @staticmethod
-    def stoploss_space() -> List[Dimension]:
-        # Override boundaries for stoploss
-        return [
-            Real(-0.33, -0.01, name='stoploss'),
-        ]
-```
-
-and then quickly switch between hyperopt classes, running optimization process with hyperopt class you need in each particular case:
-
-```
-$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy ...
-or
-$ freqtrade hyperopt --hyperopt MyAwesomeHyperOpt2 --hyperopt-loss SharpeHyperOptLossDaily --strategy MyAwesomeStrategy ...
-```
-
-## Sharing methods with your strategy
-
-Hyperopt classes provide access to the Strategy via the `strategy` class attribute.
-This can be a great way to reduce code duplication if used correctly, but will also complicate usage for inexperienced users.
-
-``` python
-from pandas import DataFrame
-from freqtrade.strategy.interface import IStrategy
-import freqtrade.vendor.qtpylib.indicators as qtpylib
-
-class MyAwesomeStrategy(IStrategy):
-
-    buy_params = {
-        'rsi-value': 30,
-        'adx-value': 35,
-    }
-
-    def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
-        return self.buy_strategy_generator(self.buy_params, dataframe, metadata)
-
-    @staticmethod
-    def buy_strategy_generator(params, dataframe: DataFrame, metadata: dict) -> DataFrame:
-        dataframe.loc[
-            (
-                qtpylib.crossed_above(dataframe['rsi'], params['rsi-value']) &
-                dataframe['adx'] > params['adx-value']) &
-                dataframe['volume'] > 0
-            )
-            , 'buy'] = 1
-        return dataframe
-
-class MyAwesomeHyperOpt(IHyperOpt):
-    ...
-    @staticmethod
-    def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
-        """
-        Define the buy strategy parameters to be used by Hyperopt.
-        """
-        def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
-            # Call strategy's buy strategy generator
-            return self.StrategyClass.buy_strategy_generator(params, dataframe, metadata)
-
-        return populate_buy_trend
-```
-
 ## Creating and using a custom loss function
 
 To use a custom loss function class, make sure that the function `hyperopt_loss_function` is defined in your custom hyperopt loss class.
@@ -142,3 +69,315 @@ This function needs to return a floating point number (`float`). Smaller numbers
 
 !!! Note
     Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface later.
+
+## Overriding pre-defined spaces
+
+To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_space`, `trailing_space`), define a nested class called Hyperopt and define the required spaces as follows:
+
+```python
+class MyAwesomeStrategy(IStrategy):
+    class HyperOpt:
+        # Define a custom stoploss space.
+        def stoploss_space(self):
+            return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')]
+```
+
+## Space options
+
+For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types:
+
+* `Categorical` - Pick from a list of categories (e.g. `Categorical(['a', 'b', 'c'], name="cat")`)
+* `Integer` - Pick from a range of whole numbers (e.g. `Integer(1, 10, name='rsi')`)
+* `SKDecimal` - Pick from a range of decimal numbers with limited precision (e.g. `SKDecimal(0.1, 0.5, decimals=3, name='adx')`). *Available only with freqtrade*.
+* `Real` - Pick from a range of decimal numbers with full precision (e.g. `Real(0.1, 0.5, name='adx')`
+
+You can import all of these from `freqtrade.optimize.space`, although `Categorical`, `Integer` and `Real` are only aliases for their corresponding scikit-optimize Spaces. `SKDecimal` is provided by freqtrade for faster optimizations.
+
+``` python
+from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real  # noqa
+```
+
+!!! Hint "SKDecimal vs. Real"
+    We recommend to use `SKDecimal` instead of the `Real` space in almost all cases. While the Real space provides full accuracy (up to ~16 decimal places) - this precision is rarely needed, and leads to unnecessary long hyperopt times.
+
+    Assuming the definition of a rather small space (`SKDecimal(0.10, 0.15, decimals=2, name='xxx')`) - SKDecimal will have 5 possibilities (`[0.10, 0.11, 0.12, 0.13, 0.14, 0.15]`).
+
+    A corresponding real space `Real(0.10, 0.15 name='xxx')`  on the other hand has an almost unlimited number of possibilities (`[0.10, 0.010000000001, 0.010000000002, ... 0.014999999999, 0.01500000000]`).
+
+---
+
+## Legacy Hyperopt
+
+This Section explains the configuration of an explicit Hyperopt file (separate to the strategy).
+
+!!! Warning "Deprecated / legacy mode"
+    Since the 2021.4 release you no longer have to write a separate hyperopt class, but all strategies can be hyperopted.
+    Please read the [main hyperopt page](hyperopt.md) for more details.
+
+### Prepare hyperopt file
+
+Configuring an explicit hyperopt file is similar to writing your own strategy, and many tasks will be similar.
+
+!!! Tip "About this page"
+    For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class.
+
+#### Create a Custom Hyperopt File
+
+The simplest way to get started is to use the following command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`.
+
+Let assume you want a hyperopt file `AwesomeHyperopt.py`:
+
+``` bash
+freqtrade new-hyperopt --hyperopt AwesomeHyperopt
+```
+
+#### Legacy Hyperopt checklist
+
+Checklist on all tasks / possibilities in hyperopt
+
+Depending on the space you want to optimize, only some of the below are required:
+
+* fill `buy_strategy_generator` - for buy signal optimization
+* fill `indicator_space` - for buy signal optimization
+* fill `sell_strategy_generator` - for sell signal optimization
+* fill `sell_indicator_space` - for sell signal optimization
+
+!!! Note
+    `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work.
+
+Optional in hyperopt - can also be loaded from a strategy (recommended):
+
+* `populate_indicators` - fallback to create indicators
+* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy
+* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy
+
+!!! Note
+    You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods.
+    Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead.
+
+Rarely you may also need to override:
+
+* `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default)
+* `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps)
+* `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default)
+* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default)
+
+#### Defining a buy signal optimization
+
+Let's say you are curious: should you use MACD crossings or lower Bollinger
+Bands to trigger your buys. And you also wonder should you use RSI or ADX to
+help with those buy decisions. If you decide to use RSI or ADX, which values
+should I use for them? So let's use hyperparameter optimization to solve this
+mystery.
+
+We will start by defining a search space:
+
+```python
+    def indicator_space() -> List[Dimension]:
+        """
+        Define your Hyperopt space for searching strategy parameters
+        """
+        return [
+            Integer(20, 40, name='adx-value'),
+            Integer(20, 40, name='rsi-value'),
+            Categorical([True, False], name='adx-enabled'),
+            Categorical([True, False], name='rsi-enabled'),
+            Categorical(['bb_lower', 'macd_cross_signal'], name='trigger')
+        ]
+```
+
+Above definition says: I have five parameters I want you to randomly combine
+to find the best combination. Two of them are integer values (`adx-value` and `rsi-value`) and I want you test in the range of values 20 to 40.  
+Then we have three category variables. First two are either `True` or `False`.
+We use these to either enable or disable the ADX and RSI guards.
+The last one we call `trigger` and use it to decide which buy trigger we want to use.
+
+So let's write the buy strategy generator using these values:
+
+```python
+    @staticmethod
+    def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
+        """
+        Define the buy strategy parameters to be used by Hyperopt.
+        """
+        def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
+            conditions = []
+            # GUARDS AND TRENDS
+            if 'adx-enabled' in params and params['adx-enabled']:
+                conditions.append(dataframe['adx'] > params['adx-value'])
+            if 'rsi-enabled' in params and params['rsi-enabled']:
+                conditions.append(dataframe['rsi'] < params['rsi-value'])
+
+            # TRIGGERS
+            if 'trigger' in params:
+                if params['trigger'] == 'bb_lower':
+                    conditions.append(dataframe['close'] < dataframe['bb_lowerband'])
+                if params['trigger'] == 'macd_cross_signal':
+                    conditions.append(qtpylib.crossed_above(
+                        dataframe['macd'], dataframe['macdsignal']
+                    ))
+
+            # Check that volume is not 0
+            conditions.append(dataframe['volume'] > 0)
+
+            if conditions:
+                dataframe.loc[
+                    reduce(lambda x, y: x & y, conditions),
+                    'buy'] = 1
+
+            return dataframe
+
+        return populate_buy_trend
+```
+
+Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations.  
+It will use the given historical data and make buys based on the buy signals generated with the above function.  
+Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)).
+
+!!! Note
+    The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators.
+    When you want to test an indicator that isn't used by the bot currently, remember to
+    add it to the `populate_indicators()` method in your strategy or hyperopt file.
+
+#### Sell optimization
+
+Similar to the buy-signal above, sell-signals can also be optimized.
+Place the corresponding settings into the following methods
+
+* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing.
+* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters.
+
+The configuration and rules are the same than for buy signals.
+To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`.
+
+### Execute Hyperopt
+
+Once you have updated your hyperopt configuration you can run it.
+Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. More time usually results in better results.
+
+We strongly recommend to use `screen` or `tmux` to prevent any connection loss.
+
+```bash
+freqtrade hyperopt --config config.json --hyperopt <hyperoptname> --hyperopt-loss <hyperoptlossname> --strategy <strategyname> -e 500 --spaces all
+```
+
+Use `<hyperoptname>` as the name of the custom hyperopt used.
+
+The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs.  
+Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results.
+
+The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below.
+
+!!! Note
+    Hyperopt will store hyperopt results with the timestamp of the hyperopt start time.
+    Reading commands (`hyperopt-list`, `hyperopt-show`) can use `--hyperopt-filename <filename>` to read and display older hyperopt results.
+    You can find a list of filenames with `ls -l user_data/hyperopt_results/`.
+
+#### Running Hyperopt using methods from a strategy
+
+Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided.
+
+```bash
+freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy
+```
+
+### Understand the Hyperopt Result
+
+Once Hyperopt is completed you can use the result to create a new strategy.
+Given the following result from hyperopt:
+
+```
+Best result:
+
+    44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
+
+Buy hyperspace params:
+{    'adx-value': 44,
+     'rsi-value': 29,
+     'adx-enabled': False,
+     'rsi-enabled': True,
+     'trigger': 'bb_lower'}
+```
+
+You should understand this result like:
+
+* The buy trigger that worked best was `bb_lower`.
+* You should not use ADX because `adx-enabled: False`)
+* You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`)
+
+You have to look inside your strategy file into `buy_strategy_generator()`
+method, what those values match to.
+
+So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block:
+
+```python
+(dataframe['rsi'] < 29.0)
+```
+
+Translating your whole hyperopt result as the new buy-signal would then look like:
+
+```python
+def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
+    dataframe.loc[
+        (
+            (dataframe['rsi'] < 29.0) &  # rsi-value
+            dataframe['close'] < dataframe['bb_lowerband']  # trigger
+        ),
+        'buy'] = 1
+    return dataframe
+```
+
+### Validate backtesting results
+
+Once the optimized parameters and conditions have been implemented into your strategy, you should backtest the strategy to make sure everything is working as expected.
+
+To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting.
+
+Should results don't match, please double-check to make sure you transferred all conditions correctly.
+Pay special care to the stoploss (and trailing stoploss) parameters, as these are often set in configuration files, which override changes to the strategy.
+You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss` or `trailing_stop`).
+
+### Sharing methods with your strategy
+
+Hyperopt classes provide access to the Strategy via the `strategy` class attribute.
+This can be a great way to reduce code duplication if used correctly, but will also complicate usage for inexperienced users.
+
+``` python
+from pandas import DataFrame
+from freqtrade.strategy.interface import IStrategy
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+class MyAwesomeStrategy(IStrategy):
+
+    buy_params = {
+        'rsi-value': 30,
+        'adx-value': 35,
+    }
+
+    def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        return self.buy_strategy_generator(self.buy_params, dataframe, metadata)
+
+    @staticmethod
+    def buy_strategy_generator(params, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        dataframe.loc[
+            (
+                qtpylib.crossed_above(dataframe['rsi'], params['rsi-value']) &
+                dataframe['adx'] > params['adx-value']) &
+                dataframe['volume'] > 0
+            )
+            , 'buy'] = 1
+        return dataframe
+
+class MyAwesomeHyperOpt(IHyperOpt):
+    ...
+    @staticmethod
+    def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
+        """
+        Define the buy strategy parameters to be used by Hyperopt.
+        """
+        def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
+            # Call strategy's buy strategy generator
+            return self.StrategyClass.buy_strategy_generator(params, dataframe, metadata)
+
+        return populate_buy_trend
+```

docs/assets/ccxt-logo.svg

@@ -0,0 +1,3 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid meet" viewBox="0 0 90 90" width="100" height="100"><defs><path d="M0 90L0 0L90 0L90 90L0 90ZM50 60L60 60L60 80L70 80L70 60L80 60L80 50L50 50L50 60ZM30 80L40 80L40 70L30 70L30 80ZM30 60L20 60L20 70L10 70L10 80L20 80L20 70L30 70L30 60L40 60L40 50L30 50L30 60ZM10 60L20 60L20 50L10 50L10 60ZM10 40L40 40L40 30L20 30L20 20L40 20L40 10L10 10L10 40ZM50 40L80 40L80 30L60 30L60 20L80 20L80 10L50 10L50 40Z" id="c6g67PWSoP"></path></defs><g><g><g><use xlink:href="#c6g67PWSoP" opacity="1" fill="#000000" fill-opacity="1"></use></g></g></g></svg>

docs/backtesting.md

@@ -15,7 +15,8 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                              [--data-format-ohlcv {json,jsongz,hdf5}]
                              [--max-open-trades INT]
                              [--stake-amount STAKE_AMOUNT] [--fee FLOAT]
-                             [--eps] [--dmmp] [--enable-protections]
+                             [-p PAIRS [PAIRS ...]] [--eps] [--dmmp]
+                             [--enable-protections]
                              [--dry-run-wallet DRY_RUN_WALLET]
                              [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]]
                              [--export EXPORT] [--export-filename PATH]
@@ -23,8 +24,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH]
 optional arguments:
   -h, --help            show this help message and exit
   -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
-                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
-                        `1d`).
+                        Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
   --data-format-ohlcv {json,jsongz,hdf5}
@@ -38,6 +38,9 @@ optional arguments:
                         setting.
   --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
                         entry and exit).
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Limit command to these pairs. Pairs are space-
+                        separated.
   --eps, --enable-position-stacking
                         Allow buying the same pair multiple times (position
                         stacking).
@@ -421,6 +424,7 @@ It contains some useful key metrics about performance of your strategy on backte
 Since backtesting lacks some detailed information about what happens within a candle, it needs to take a few assumptions:
 
 - Buys happen at open-price
+- All orders are filled at the requested price (no slippage, no unfilled orders)
 - Sell-signal sells happen at open-price of the consecutive candle
 - Sell-signal is favored over Stoploss, because sell-signals are assumed to trigger on candle's open
 - ROI

docs/configuration.md

@@ -11,7 +11,16 @@ Per default, the bot loads the configuration from the `config.json` file, locate
 
 You can specify a different configuration file used by the bot with the `-c/--config` command line option.
 
-In some advanced use cases, multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
+Multiple configuration files can be specified and used by the bot or the bot can read its configuration parameters from the process standard input stream.
+
+!!! Tip "Use multiple configuration files to keep secrets secret"
+    You can use a 2nd configuration file containing your secrets. That way you can share your "primary" configuration file, while still keeping your API keys for yourself.
+
+    ``` bash
+    freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>
+    ```
+    The 2nd file should only specify what you intend to override.
+    If a key is in more than one of the configurations, then the "last specified configuration" wins (in the above example, `config-private.json`).
 
 If you used the [Quick start](installation.md/#quick-start) method for installing 
 the bot, the installation script should have already created the default configuration file (`config.json`) for you.
@@ -167,7 +176,7 @@ This exchange has also a limit on USD - where all orders must be > 10$ - which h
 
 To guarantee safe execution, freqtrade will not allow buying with a stake-amount of 10.1$, instead, it'll make sure that there's enough space to place a stoploss below the pair (+ an offset, defined by `amount_reserve_percent`, which defaults to 5%).
 
-With a stoploss of 10% - we'd therefore end up with a value of ~13.8$ (`12 * (1 + 0.05 + 0.1)`).
+With a reserve of 5%, the minimum stake amount would be ~12.6$ (`12 * (1 + 0.05)`). If we take in account a stoploss of 10% on top of that - we'd end up with a value of ~14$ (`12.6 / (1 - 0.1)`).
 
 To limit this calculation in case of large stoploss values, the calculated minimum stake-limit will never be more than 50% above the real limit.
 
@@ -518,16 +527,27 @@ API Keys are usually only required for live trading (trading for real money, bot
 **Insert your Exchange API key (change them by fake api keys):**
 
 ```json
-"exchange": {
+{
+    "exchange": {
         "name": "bittrex",
         "key": "af8ddd35195e9dc500b9a6f799f6f5c93d89193b",
         "secret": "08a9dc6db3d7b53e1acebd9275677f4b0a04f1a5",
-        ...
+        //"password": "", // Optional, not needed by all exchanges)
+        // ...
+    }
+    //...
 }
 ```
 
 You should also make sure to read the [Exchanges](exchanges.md) section of the documentation to be aware of potential configuration details specific to your exchange.
 
+!!! Hint "Keep your secrets secret"
+    To keep your secrets secret, we recommend to use a 2nd configuration for your API keys.
+    Simply use the above snippet in a new configuration file (e.g. `config-private.json`) and keep your settings in this file.
+    You can then start the bot with `freqtrade trade --config user_data/config.json --config user_data/config-private.json <...>` to have your keys loaded.
+
+    **NEVER** share your private configuration file or your exchange keys with anyone!
+
 ### Using proxy with Freqtrade
 
 To use a proxy with freqtrade, add the kwarg `"aiohttp_trust_env"=true` to the `"ccxt_async_kwargs"` dict in the exchange section of the configuration.

docs/data-download.md

@@ -11,8 +11,9 @@ Otherwise `--exchange` becomes mandatory.
 You can use a relative timerange (`--days 20`) or an absolute starting point (`--timerange 20200101-`). For incremental downloads, the relative approach should be used.
 
 !!! Tip "Tip: Updating existing data"
-    If you already have backtesting data available in your data-directory and would like to refresh this data up to today, use `--days xx` with a number slightly higher than the missing number of days. Freqtrade will keep the available data and only download the missing data.
-    Be careful though: If the number is too small (which would result in a few missing days), the whole dataset will be removed and only xx days will be downloaded.
+    If you already have backtesting data available in your data-directory and would like to refresh this data up to today, do not use `--days` or `--timerange` parameters. Freqtrade will keep the available data and only download the missing data.
+    If you are updating existing data after inserting new pairs that you have no data for, use `--new-pairs-days xx` parameter. Specified number of days will be downloaded for new pairs while old pairs will be updated with missing data only.
+    If you use `--days xx` parameter alone - data for specified number of days will be downloaded for _all_ pairs. Be careful, if specified number of days is smaller than gap between now and last downloaded candle - freqtrade will delete all existing data to avoid gaps in candle data.
 
 ### Usage
 
@@ -20,8 +21,9 @@ You can use a relative timerange (`--days 20`) or an absolute starting point (`-
 usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
                                [-d PATH] [--userdir PATH]
                                [-p PAIRS [PAIRS ...]] [--pairs-file FILE]
-                               [--days INT] [--timerange TIMERANGE]
-                               [--dl-trades] [--exchange EXCHANGE]
+                               [--days INT] [--new-pairs-days INT]
+                               [--timerange TIMERANGE] [--dl-trades]
+                               [--exchange EXCHANGE]
                                [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
                                [--erase]
                                [--data-format-ohlcv {json,jsongz,hdf5}]
@@ -30,10 +32,12 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
-                        Show profits for only these pairs. Pairs are space-
+                        Limit command to these pairs. Pairs are space-
                         separated.
   --pairs-file FILE     File containing a list of pairs to download.
   --days INT            Download data for given number of days.
+  --new-pairs-days INT  Download data of new pairs for given number of days.
+                        Default: `None`.
   --timerange TIMERANGE
                         Specify what timerange of data to use.
   --dl-trades           Download trades instead of OHLCV data. The bot will
@@ -48,10 +52,10 @@ optional arguments:
                         exchange/pairs/timeframes.
   --data-format-ohlcv {json,jsongz,hdf5}
                         Storage format for downloaded candle (OHLCV) data.
-                        (default: `json`).
+                        (default: `None`).
   --data-format-trades {json,jsongz,hdf5}
                         Storage format for downloaded trades data. (default:
-                        `jsongz`).
+                        `None`).
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).

docs/docker_quickstart.md

@@ -14,7 +14,7 @@ To simplify running freqtrade, please install [`docker-compose`](https://docs.do
 
 ## Freqtrade with docker-compose
 
-Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) ready for usage.
+Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.com/r/freqtradeorg/freqtrade/), as well as a [docker-compose file](https://github.com/freqtrade/freqtrade/blob/stable/docker-compose.yml) ready for usage.
 
 !!! Note
     - The following section assumes that `docker` and `docker-compose` are installed and available to the logged in user.
@@ -22,7 +22,7 @@ Freqtrade provides an official Docker image on [Dockerhub](https://hub.docker.co
 
 ### Docker quick start
 
-Create a new directory and place the [docker-compose file](https://github.com/freqtrade/freqtrade/blob/develop/docker-compose.yml) in this directory.
+Create a new directory and place the [docker-compose file](https://raw.githubusercontent.com/freqtrade/freqtrade/stable/docker-compose.yml) in this directory.
 
 === "PC/MAC/Linux"
     ``` bash
@@ -156,8 +156,8 @@ Head over to the [Backtesting Documentation](backtesting.md) to learn more.
 
 ### Additional dependencies with docker-compose
 
-If your strategy requires dependencies not included in the default image (like [technical](https://github.com/freqtrade/technical)) - it will be necessary to build the image on your host.
-For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) for an example).
+If your strategy requires dependencies not included in the default image - it will be necessary to build the image on your host.
+For this, please create a Dockerfile containing installation steps for the additional dependencies (have a look at [docker/Dockerfile.custom](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.custom) for an example).
 
 You'll then also need to modify the `docker-compose.yml` file and uncomment the build step, as well as rename the image to avoid naming collisions.
 

docs/edge.md

@@ -1,9 +1,9 @@
 # Edge positioning
 
-The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss. 
+The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss.
 
 !!! Warning
-    `Edge positioning` is not compatible with dynamic (volume-based) whitelist.
+    WHen using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data.
 
 !!! Note
     `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file.
@@ -14,7 +14,7 @@ The `Edge Positioning` module uses probability to calculate your win rate and ri
 
 Trading strategies are not perfect. They are frameworks that are susceptible to the market and its indicators. Because the market is not at all predictable, sometimes a strategy will win and sometimes the same strategy will lose.
 
-To obtain an edge in the market, a strategy has to make more money than it loses. Making money in trading is not only about *how often* the strategy makes or loses money. 
+To obtain an edge in the market, a strategy has to make more money than it loses. Making money in trading is not only about *how often* the strategy makes or loses money.
 
 !!! tip "It doesn't matter how often, but how much!"
     A bad strategy might make 1 penny in *ten* transactions but lose 1 dollar in *one* transaction. If one only checks the number of winning trades, it would be misleading to think that the strategy is actually making a profit.
@@ -215,16 +215,20 @@ Let's say the stake currency is **ETH** and there is $10$ **ETH** on the wallet.
 usage: freqtrade edge [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                       [--userdir PATH] [-s NAME] [--strategy-path PATH]
                       [-i TIMEFRAME] [--timerange TIMERANGE]
+                      [--data-format-ohlcv {json,jsongz,hdf5}]
                       [--max-open-trades INT] [--stake-amount STAKE_AMOUNT]
-                      [--fee FLOAT] [--stoplosses STOPLOSS_RANGE]
+                      [--fee FLOAT] [-p PAIRS [PAIRS ...]]
+                      [--stoplosses STOPLOSS_RANGE]
 
 optional arguments:
   -h, --help            show this help message and exit
   -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
-                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
-                        `1d`).
+                        Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
+  --data-format-ohlcv {json,jsongz,hdf5}
+                        Storage format for downloaded candle (OHLCV) data.
+                        (default: `None`).
   --max-open-trades INT
                         Override the value of the `max_open_trades`
                         configuration setting.
@@ -233,6 +237,9 @@ optional arguments:
                         setting.
   --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
                         entry and exit).
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Limit command to these pairs. Pairs are space-
+                        separated.
   --stoplosses STOPLOSS_RANGE
                         Defines a range of stoploss values against which edge
                         will assess the strategy. The format is "min,max,step"

docs/exchanges.md

@@ -7,10 +7,10 @@ This page combines common gotchas and informations which are exchange-specific a
 !!! Tip "Stoploss on Exchange"
     Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it.
 
-### Blacklists
+### Binance Blacklist
 
 For Binance, please add `"BNB/<STAKE>"` to your blacklist to avoid issues.
-Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB order unsellable as the expected amount is not there anymore.
+Accounts having BNB accounts use this to pay for fees - if your first trade happens to be on `BNB`, further trades will consume this position and make the initial BNB trade unsellable as the expected amount is not there anymore.
 
 ### Binance sites
 
@@ -44,6 +44,10 @@ Due to the heavy rate-limiting applied by Kraken, the following configuration se
     Downloading kraken data will require significantly more memory (RAM) than any other exchange, as the trades-data needs to be converted into candles on your machine.
     It will also take a long time, as freqtrade will need to download every single trade that happened on the exchange for the pair / timerange combination, therefore please be patient.
 
+!!! Warning "rateLimit tuning"
+    Please pay attention that rateLimit configuration entry holds delay in milliseconds between requests, NOT requests\sec rate.
+    So, in order to mitigate Kraken API "Rate limit exceeded" exception, this configuration should be increased, NOT decreased.
+
 ## Bittrex
 
 ### Order types
@@ -96,6 +100,23 @@ To use subaccounts with FTX, you need to edit the configuration and add the foll
 }
 ```
 
+## Kucoin
+
+Kucoin requries a passphrase for each api key, you will therefore need to add this key into the configuration so your exchange section looks as follows:
+
+```json
+"exchange": {
+    "name": "kucoin",
+    "key": "your_exchange_key",
+    "secret": "your_exchange_secret",
+    "password": "your_exchange_api_key_password",
+```
+
+### Kucoin Blacklists
+
+For Kucoin, please add `"KCS/<STAKE>"` to your blacklist to avoid issues.
+Accounts having KCS accounts use this to pay for fees - if your first trade happens to be on `KCS`, further trades will consume this position and make the initial KCS trade unsellable as the expected amount is not there anymore.
+
 ## All exchanges
 
 Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys.

docs/faq.md

@@ -1,5 +1,19 @@
 # Freqtrade FAQ
 
+## Supported Markets
+
+Freqtrade supports spot trading only.
+
+### Can I open short positions?
+
+No, Freqtrade does not support trading with margin / leverage, and cannot open short positions.
+
+In some cases, your exchange may provide leveraged spot tokens which can be traded with Freqtrade eg. BTCUP/USD, BTCDOWN/USD, ETHBULL/USD, ETHBEAR/USD, etc...
+
+### Can I trade options or futures?
+
+No, options and futures trading are not supported.
+
 ## Beginner Tips & Tricks
 
 * When you work with your strategy & hyperopt file you should use a proper code editor like VSCode or PyCharm. A good code editor will provide syntax highlighting as well as line numbers, making it easy to find syntax errors (most likely pointed out by Freqtrade during startup).
@@ -142,7 +156,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD
 
 ### Why does it take a long time to run hyperopt?
 
-* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/X89cVG). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
+* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/MA9v74M). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you.
 
 * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers:
 

docs/hyperopt.md

@@ -1,19 +1,22 @@
 # Hyperopt
 
 This page explains how to tune your strategy by finding the optimal
-parameters, a process called hyperparameter optimization. The bot uses several
-algorithms included in the `scikit-optimize` package to accomplish this. The
-search will burn all your CPU cores, make your laptop sound like a fighter jet
-and still take a long time.
+parameters, a process called hyperparameter optimization. The bot uses algorithms included in the `scikit-optimize` package to accomplish this.
+The search will burn all your CPU cores, make your laptop sound like a fighter jet and still take a long time.
 
 In general, the search for best parameters starts with a few random combinations (see [below](#reproducible-results) for more details) and then uses Bayesian search with a ML regressor algorithm (currently ExtraTreesRegressor) to quickly find a combination of parameters in the search hyperspace that minimizes the value of the [loss function](#loss-functions).
 
-Hyperopt requires historic data to be available, just as backtesting does.
+Hyperopt requires historic data to be available, just as backtesting does (hyperopt runs backtesting many times with different parameters).
 To learn how to get data for the pairs and exchange you're interested in, head over to the [Data Downloading](data-download.md) section of the documentation.
 
 !!! Bug
     Hyperopt can crash when used with only 1 CPU Core as found out in [Issue #1133](https://github.com/freqtrade/freqtrade/issues/1133)
 
+!!! Note
+    Since 2021.4 release you no longer have to write a separate hyperopt class, but can configure the parameters directly in the strategy.
+    The legacy method is still supported, but it is no longer the recommended way of setting up hyperopt. 
+    The legacy documentation is available at [Legacy Hyperopt](advanced-hyperopt.md#legacy-hyperopt).
+
 ## Install hyperopt dependencies
 
 Since Hyperopt dependencies are not needed to run the bot itself, are heavy, can not be easily built on some platforms (like Raspberry PI), they are not installed by default. Before you run Hyperopt, you need to install the corresponding dependencies, as described in this section below.
@@ -34,7 +37,6 @@ pip install -r requirements-hyperopt.txt
 
 ## Hyperopt command reference
 
-
 ```
 usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                           [--userdir PATH] [-s NAME] [--strategy-path PATH]
@@ -42,8 +44,9 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
                           [--data-format-ohlcv {json,jsongz,hdf5}]
                           [--max-open-trades INT]
                           [--stake-amount STAKE_AMOUNT] [--fee FLOAT]
-                          [--hyperopt NAME] [--hyperopt-path PATH] [--eps]
-                          [--dmmp] [--enable-protections]
+                          [-p PAIRS [PAIRS ...]] [--hyperopt NAME]
+                          [--hyperopt-path PATH] [--eps] [--dmmp]
+                          [--enable-protections]
                           [--dry-run-wallet DRY_RUN_WALLET] [-e INT]
                           [--spaces {all,buy,sell,roi,stoploss,trailing,default} [{all,buy,sell,roi,stoploss,trailing,default} ...]]
                           [--print-all] [--no-color] [--print-json] [-j JOBS]
@@ -53,8 +56,7 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH]
 optional arguments:
   -h, --help            show this help message and exit
   -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
-                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
-                        `1d`).
+                        Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
   --timerange TIMERANGE
                         Specify what timerange of data to use.
   --data-format-ohlcv {json,jsongz,hdf5}
@@ -68,6 +70,9 @@ optional arguments:
                         setting.
   --fee FLOAT           Specify fee ratio. Will be applied twice (on trade
                         entry and exit).
+  -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
+                        Limit command to these pairs. Pairs are space-
+                        separated.
   --hyperopt NAME       Specify hyperopt class name which will be used by the
                         bot.
   --hyperopt-path PATH  Specify additional lookup path for Hyperopt and
@@ -104,7 +109,8 @@ optional arguments:
                         reproducible hyperopt results.
   --min-trades INT      Set minimal desired number of trades for evaluations
                         in the hyperopt optimization path (default: 1).
-  --hyperopt-loss NAME  Specify the class name of the hyperopt loss function
+  --hyperopt-loss NAME, --hyperoptloss NAME
+                        Specify the class name of the hyperopt loss function
                         class (IHyperOptLoss). Different functions can
                         generate completely different results, since the
                         target for optimization is different. Built-in
@@ -137,47 +143,19 @@ Strategy arguments:
 
 ```
 
-## Prepare Hyperopting
-
-Before we start digging into Hyperopt, we recommend you to take a look at
-the sample hyperopt file located in [user_data/hyperopts/](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt.py).
-
-Configuring hyperopt is similar to writing your own strategy, and many tasks will be similar.
-
-!!! Tip "About this page"
-    For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class.
-
-The simplest way to get started is to use the following, command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`.
-
-``` bash
-freqtrade new-hyperopt --hyperopt AwesomeHyperopt
-```
-
 ### Hyperopt checklist
 
 Checklist on all tasks / possibilities in hyperopt
 
 Depending on the space you want to optimize, only some of the below are required:
 
-* fill `buy_strategy_generator` - for buy signal optimization
-* fill `indicator_space` - for buy signal optimization
-* fill `sell_strategy_generator` - for sell signal optimization
-* fill `sell_indicator_space` - for sell signal optimization
+* define parameters with `space='buy'` - for buy signal optimization
+* define parameters with `space='sell'` - for sell signal optimization
 
 !!! Note
     `populate_indicators` needs to create all indicators any of the spaces may use, otherwise hyperopt will not work.
 
-Optional in hyperopt - can also be loaded from a strategy (recommended):
-
-* `populate_indicators` - fallback to create indicators
-* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy
-* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy
-
-!!! Note
-    You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods.
-    Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead.
-
-Rarely you may also need to override:
+Rarely you may also need to create a [nested class](advanced-hyperopt.md#overriding-pre-defined-spaces) named `HyperOpt` and implement
 
 * `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default)
 * `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps)
@@ -185,31 +163,30 @@ Rarely you may also need to override:
 * `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default)
 
 !!! Tip "Quickly optimize ROI, stoploss and trailing stoploss"
-    You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything (i.e. without creation of a "complete" Hyperopt class with dimensions, parameters, triggers and guards, as described in this document) from the default hyperopt template by relying on your strategy to do most of the calculations.
+    You can quickly optimize the spaces `roi`, `stoploss` and `trailing` without changing anything in your strategy.
 
-    ```python
+    ``` bash
     # Have a working strategy at hand.
-    freqtrade new-hyperopt --hyperopt EmptyHyperopt
-
-    freqtrade hyperopt --hyperopt EmptyHyperopt --hyperopt-loss SharpeHyperOptLossDaily --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100
+    freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --spaces roi stoploss trailing --strategy MyWorkingStrategy --config config.json -e 100
     ```
 
-### Create a Custom Hyperopt File
+### Hyperopt execution logic
 
-Let assume you want a hyperopt file `AwesomeHyperopt.py`:  
+Hyperopt will first load your data into memory and will then run `populate_indicators()` once per Pair to generate all indicators.
 
-``` bash
-freqtrade new-hyperopt --hyperopt AwesomeHyperopt
-```
+Hyperopt will then spawn into different processes (number of processors, or `-j <n>`), and run backtesting over and over again, changing the parameters that are part of the `--spaces` defined.
 
-This command will create a new hyperopt file from a template, allowing you to get started quickly.
+For every new set of parameters, freqtrade will run first `populate_buy_trend()` followed by `populate_sell_trend()`, and then run the regular backtesting process to simulate trades.
+
+After backtesting, the results are passed into the [loss function](#loss-functions), which will evaluate if this result was better or worse than previous results.  
+Based on the loss function result, hyperopt will determine the next set of parameters to try in the next round of backtesting.
 
 ### Configure your Guards and Triggers
 
-There are two places you need to change in your hyperopt file to add a new buy hyperopt for testing:
+There are two places you need to change in your strategy file to add a new buy hyperopt for testing:
 
-* Inside `indicator_space()` - the parameters hyperopt shall be optimizing.
-* Within `buy_strategy_generator()` - populate the nested `populate_buy_trend()` to apply the parameters.
+* Define the parameters at the class level hyperopt shall be optimizing.
+* Within `populate_buy_trend()` - use defined parameter values instead of raw constants.
 
 There you have two different types of indicators: 1. `guards` and 2. `triggers`.
 
@@ -221,100 +198,106 @@ There you have two different types of indicators: 1. `guards` and 2. `triggers`.
     However, this guide will make this distinction to make it clear that signals should not be "sticking".
     Sticking signals are signals that are active for multiple candles. This can lead into buying a signal late (right before the signal disappears - which means that the chance of success is a lot lower than right at the beginning).
 
-Hyper-optimization will, for each epoch round, pick one trigger and possibly
-multiple guards. The constructed strategy will be something like "*buy exactly when close price touches lower Bollinger band, BUT only if
-ADX > 10*".
-
-If you have updated the buy strategy, i.e. changed the contents of `populate_buy_trend()` method, you have to update the `guards` and `triggers` your hyperopt must use correspondingly.
+Hyper-optimization will, for each epoch round, pick one trigger and possibly multiple guards.
 
 #### Sell optimization
 
 Similar to the buy-signal above, sell-signals can also be optimized.
 Place the corresponding settings into the following methods
 
-* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing.
-* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters.
+* Define the parameters at the class level hyperopt shall be optimizing, either naming them `sell_*`, or by explicitly defining `space='sell'`.
+* Within `populate_sell_trend()` - use defined parameter values instead of raw constants.
 
 The configuration and rules are the same than for buy signals.
-To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`.
-
-#### Using timeframe as a part of the Strategy
-
-The Strategy class exposes the timeframe value as the `self.timeframe` attribute.
-The same value is available as class-attribute `HyperoptName.timeframe`.
-In the case of the linked sample-value this would be `AwesomeHyperopt.timeframe`.
 
 ## Solving a Mystery
 
-Let's say you are curious: should you use MACD crossings or lower Bollinger
-Bands to trigger your buys. And you also wonder should you use RSI or ADX to
-help with those buy decisions. If you decide to use RSI or ADX, which values
-should I use for them? So let's use hyperparameter optimization to solve this
-mystery.
+Let's say you are curious: should you use MACD crossings or lower Bollinger Bands to trigger your buys. 
+And you also wonder should you use RSI or ADX to help with those buy decisions. 
+If you decide to use RSI or ADX, which values should I use for them? 
 
-We will start by defining a search space:
+So let's use hyperparameter optimization to solve this mystery.
 
-```python
-    def indicator_space() -> List[Dimension]:
+### Defining indicators to be used
+
+We start by calculating the indicators our strategy is going to use.
+
+``` python
+class MyAwesomeStrategy(IStrategy):
+
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
         """
-        Define your Hyperopt space for searching strategy parameters
+        Generate all indicators used by the strategy
         """
-        return [
-            Integer(20, 40, name='adx-value'),
-            Integer(20, 40, name='rsi-value'),
-            Categorical([True, False], name='adx-enabled'),
-            Categorical([True, False], name='rsi-enabled'),
-            Categorical(['bb_lower', 'macd_cross_signal'], name='trigger')
-        ]
+        dataframe['adx'] = ta.ADX(dataframe)
+        dataframe['rsi'] = ta.RSI(dataframe)
+        macd = ta.MACD(dataframe)
+        dataframe['macd'] = macd['macd']
+        dataframe['macdsignal'] = macd['macdsignal']
+        dataframe['macdhist'] = macd['macdhist']
+
+        bollinger = ta.BBANDS(dataframe, timeperiod=20, nbdevup=2.0, nbdevdn=2.0)
+        dataframe['bb_lowerband'] = boll['lowerband']
+        dataframe['bb_middleband'] = boll['middleband']
+        dataframe['bb_upperband'] = boll['upperband']
+        return dataframe
 ```
 
-Above definition says: I have five parameters I want you to randomly combine
-to find the best combination. Two of them are integer values (`adx-value`
-and `rsi-value`) and I want you test in the range of values 20 to 40.  
+### Hyperoptable parameters
+
+We continue to define hyperoptable parameters:
+
+```python
+class MyAwesomeStrategy(IStrategy):
+    buy_adx = IntParameter(20, 40, default=30, space="buy")
+    buy_rsi = IntParameter(20, 40, default=30, space="buy")
+    buy_adx_enabled = CategoricalParameter([True, False], space="buy")
+    buy_rsi_enabled = CategoricalParameter([True, False], space="buy")
+    buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal'], space="buy")
+```
+
+Above definition says: I have five parameters I want to randomly combine to find the best combination.
+Two of them are integer values (`buy_adx` and `buy_rsi`) and I want you test in the range of values 20 to 40.  
 Then we have three category variables. First two are either `True` or `False`.
-We use these to either enable or disable the ADX and RSI guards. The last
-one we call `trigger` and use it to decide which buy trigger we want to use.
+We use these to either enable or disable the ADX and RSI guards.
+The last one we call `trigger` and use it to decide which buy trigger we want to use.
+
+!!! Note "Parameter space assignment"
+    Parameters must either be assigned to a variable named `buy_*` or `sell_*` - or contain `space='buy'` | `space='sell'` to be assigned to a space correctly.
+    If no parameter is available for a space, you'll receive the error that no space was found when running hyperopt.
 
 So let's write the buy strategy using these values:
 
 ```python
-    @staticmethod
-    def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
-        """
-        Define the buy strategy parameters to be used by Hyperopt.
-        """
-        def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame:
-            conditions = []
-            # GUARDS AND TRENDS
-            if 'adx-enabled' in params and params['adx-enabled']:
-                conditions.append(dataframe['adx'] > params['adx-value'])
-            if 'rsi-enabled' in params and params['rsi-enabled']:
-                conditions.append(dataframe['rsi'] < params['rsi-value'])
+    def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        conditions = []
+        # GUARDS AND TRENDS
+        if self.buy_adx_enabled.value:
+            conditions.append(dataframe['adx'] > self.buy_adx.value)
+        if self.buy_rsi_enabled.value:
+            conditions.append(dataframe['rsi'] < self.buy_rsi.value)
 
-            # TRIGGERS
-            if 'trigger' in params:
-                if params['trigger'] == 'bb_lower':
-                    conditions.append(dataframe['close'] < dataframe['bb_lowerband'])
-                if params['trigger'] == 'macd_cross_signal':
-                    conditions.append(qtpylib.crossed_above(
-                        dataframe['macd'], dataframe['macdsignal']
-                    ))
+        # TRIGGERS
+        if self.buy_trigger.value == 'bb_lower':
+            conditions.append(dataframe['close'] < dataframe['bb_lowerband'])
+        if self.buy_trigger.value == 'macd_cross_signal':
+            conditions.append(qtpylib.crossed_above(
+                dataframe['macd'], dataframe['macdsignal']
+            ))
 
-            # Check that volume is not 0
-            conditions.append(dataframe['volume'] > 0)
+        # Check that volume is not 0
+        conditions.append(dataframe['volume'] > 0)
 
-            if conditions:
-                dataframe.loc[
-                    reduce(lambda x, y: x & y, conditions),
-                    'buy'] = 1
+        if conditions:
+            dataframe.loc[
+                reduce(lambda x, y: x & y, conditions),
+                'buy'] = 1
 
-            return dataframe
-
-        return populate_buy_trend
+        return dataframe
 ```
 
 Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations.  
-It will use the given historical data and make buys based on the buy signals generated with the above function.  
+It will use the given historical data and simulate buys based on the buy signals generated with the above function.  
 Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)).
 
 !!! Note
@@ -322,6 +305,108 @@ Based on the results, hyperopt will tell you which parameter combination produce
     When you want to test an indicator that isn't used by the bot currently, remember to
     add it to the `populate_indicators()` method in your strategy or hyperopt file.
 
+## Parameter types
+
+There are four parameter types each suited for different purposes.
+
+* `IntParameter` - defines an integral parameter with upper and lower boundaries of search space.
+* `DecimalParameter` - defines a floating point parameter with a limited number of decimals (default 3). Should be preferred instead of `RealParameter` in most cases.
+* `RealParameter` - defines a floating point parameter with upper and lower boundaries and no precision limit. Rarely used as it creates a space with a near infinite number of possibilities.
+* `CategoricalParameter` - defines a parameter with a predetermined number of choices.
+
+!!! Tip "Disabling parameter optimization"
+    Each parameter takes two boolean parameters:
+    * `load` - when set to `False` it will not load values configured in `buy_params` and `sell_params`.
+    * `optimize` - when set to `False` parameter will not be included in optimization process.
+    Use these parameters to quickly prototype various ideas.
+
+!!! Warning
+    Hyperoptable parameters cannot be used in `populate_indicators` - as hyperopt does not recalculate indicators for each epoch, so the starting value would be used in this case.
+
+### Optimizing an indicator parameter
+
+Assuming you have a simple strategy in mind - a EMA cross strategy (2 Moving averages crossing) - and you'd like to find the ideal parameters for this strategy.
+
+``` python
+from pandas import DataFrame
+from functools import reduce
+
+import talib.abstract as ta
+
+from freqtrade.strategy import IStrategy
+from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter
+import freqtrade.vendor.qtpylib.indicators as qtpylib
+
+class MyAwesomeStrategy(IStrategy):
+    stoploss = -0.05
+    timeframe = '15m'
+    # Define the parameter spaces
+    buy_ema_short = IntParameter(3, 50, default=5)
+    buy_ema_long = IntParameter(15, 200, default=50)
+
+
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        """Generate all indicators used by the strategy"""
+        
+        # Calculate all ema_short values
+        for val in self.buy_ema_short.range:
+            dataframe[f'ema_short_{val}'] = ta.EMA(dataframe, timeperiod=val)
+        
+        # Calculate all ema_long values
+        for val in self.buy_ema_long.range:
+            dataframe[f'ema_long_{val}'] = ta.EMA(dataframe, timeperiod=val)
+        
+        return dataframe
+
+    def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        conditions = []
+        conditions.append(qtpylib.crossed_above(
+                dataframe[f'ema_short_{self.buy_ema_short.value}'], dataframe[f'ema_long_{self.buy_ema_long.value}']
+            ))
+
+        # Check that volume is not 0
+        conditions.append(dataframe['volume'] > 0)
+
+        if conditions:
+            dataframe.loc[
+                reduce(lambda x, y: x & y, conditions),
+                'buy'] = 1
+        return dataframe
+
+    def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        conditions = []
+        conditions.append(qtpylib.crossed_above(
+                dataframe[f'ema_long_{self.buy_ema_long.value}'], dataframe[f'ema_short_{self.buy_ema_short.value}']
+            ))
+
+        # Check that volume is not 0
+        conditions.append(dataframe['volume'] > 0)
+
+        if conditions:
+            dataframe.loc[
+                reduce(lambda x, y: x & y, conditions),
+                'sell'] = 1
+        return dataframe
+```
+
+Breaking it down:
+
+Using `self.buy_ema_short.range` will return a range object containing all entries between the Parameters low and high value.
+In this case (`IntParameter(3, 50, default=5)`), the loop would run for all numbers between 3 and 50 (`[3, 4, 5, ... 49, 50]`).
+By using this in a loop, hyperopt will generate 48 new columns (`['buy_ema_3', 'buy_ema_4', ... , 'buy_ema_50']`).
+
+Hyperopt itself will then use the selected value to create the buy and sell signals
+
+While this strategy is most likely too simple to provide consistent profit, it should serve as an example how optimize indicator parameters.
+
+!!! Note
+    `self.buy_ema_short.range` will act differently between hyperopt and other modes. For hyperopt, the above example may generate 48 new columns, however for all other modes (backtesting, dry/live), it will only generate the column for the selected value. You should therefore avoid using the resulting column with explicit values (values other than `self.buy_ema_short.value`).
+
+??? Hint "Performance tip"
+    By doing the calculation of all possible indicators in `populate_indicators()`, the calculation of the indicator happens only once for every parameter.  
+    While this may slow down the hyperopt startup speed, the overall performance will increase as the Hyperopt execution itself may pick the same value for multiple epochs (changing other values).
+    You should however try to use space ranges as small as possible. Every new column will require more memory, and every possibility hyperopt can try will increase the search space.
+
 ## Loss-functions
 
 Each hyperparameter tuning requires a target. This is usually defined as a loss function (sometimes also called objective function), which should decrease for more desirable results, and increase for bad results.
@@ -343,16 +428,14 @@ Creation of a custom loss function is covered in the [Advanced Hyperopt](advance
 ## Execute Hyperopt
 
 Once you have updated your hyperopt configuration you can run it.
-Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. More time usually results in better results.
+Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result.
 
 We strongly recommend to use `screen` or `tmux` to prevent any connection loss.
 
 ```bash
-freqtrade hyperopt --config config.json --hyperopt <hyperoptname> --hyperopt-loss <hyperoptlossname> --strategy <strategyname> -e 500 --spaces all
+freqtrade hyperopt --config config.json --hyperopt-loss <hyperoptlossname> --strategy <strategyname> -e 500 --spaces all
 ```
 
-Use `<hyperoptname>` as the name of the custom hyperopt used.
-
 The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs.  
 Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results.
 
@@ -366,30 +449,23 @@ The `--spaces all` option determines that all possible parameters should be opti
 ### Execute Hyperopt with different historical data source
 
 If you would like to hyperopt parameters using an alternate historical data set that
-you have on-disk, use the `--datadir PATH` option. By default, hyperopt
-uses data from directory `user_data/data`.
+you have on-disk, use the `--datadir PATH` option. By default, hyperopt uses data from directory `user_data/data`.
 
 ### Running Hyperopt with a smaller test-set
 
 Use the `--timerange` argument to change how much of the test-set you want to use.
-For example, to use one month of data, pass the following parameter to the hyperopt call:
+For example, to use one month of data, pass `--timerange 20210101-20210201` (from january 2021 - february 2021) to the hyperopt call.
+
+Full command:
 
 ```bash
-freqtrade hyperopt --hyperopt <hyperoptname> --strategy <strategyname> --timerange 20180401-20180501
-```
-
-### Running Hyperopt using methods from a strategy
-
-Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided.
-
-```bash
-freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy
+freqtrade hyperopt --hyperopt <hyperoptname> --strategy <strategyname> --timerange 20210101-20210201
 ```
 
 ### Running Hyperopt with Smaller Search Space
 
 Use the `--spaces` option to limit the search space used by hyperopt.
-Letting Hyperopt optimize everything is a huuuuge search space. 
+Letting Hyperopt optimize everything is a huuuuge search space.
 Often it might make more sense to start by just searching for initial buy algorithm.
 Or maybe you just want to optimize your stoploss or roi table for that awesome new buy strategy you have.
 
@@ -406,40 +482,9 @@ Legal values are:
 
 The default Hyperopt Search Space, used when no `--space` command line option is specified, does not include the `trailing` hyperspace. We recommend you to run optimization for the `trailing` hyperspace separately, when the best parameters for other hyperspaces were found, validated and pasted into your custom strategy.
 
-### Position stacking and disabling max market positions
-
-In some situations, you may need to run Hyperopt (and Backtesting) with the
-`--eps`/`--enable-position-staking` and `--dmmp`/`--disable-max-market-positions` arguments.
-
-By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one
-open trade is allowed for every traded pair. The total number of trades open for all pairs
-is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to
-some potential trades to be hidden (or masked) by previously open trades.
-
-The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times,
-while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades`
-during Hyperopt/Backtesting (which is equal to setting `max_open_trades` to a very high
-number).
-
-!!! Note
-    Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality.
-
-You can also enable position stacking in the configuration file by explicitly setting
-`"position_stacking"=true`.
-
-### Reproducible results
-
-The search for optimal parameters starts with a few (currently 30) random combinations in the hyperspace of parameters, random Hyperopt epochs. These random epochs are marked with an asterisk character (`*`) in the first column in the Hyperopt output.
-
-The initial state for generation of these random values (random state) is controlled by the value of the `--random-state` command line option. You can set it to some arbitrary value of your choice to obtain reproducible results.
-
-If you have not set this value explicitly in the command line options, Hyperopt seeds the random state with some random value for you. The random state value for each Hyperopt run is shown in the log, so you can copy and paste it into the `--random-state` command line option to repeat the set of the initial random epochs used.
-
-If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyper-optimization results with same random state value used.
-
 ## Understand the Hyperopt Result
 
-Once Hyperopt is completed you can use the result to create a new strategy.
+Once Hyperopt is completed you can use the result to update your strategy.
 Given the following result from hyperopt:
 
 ```
@@ -447,49 +492,38 @@ Best result:
 
     44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
 
-Buy hyperspace params:
-{    'adx-value': 44,
-     'rsi-value': 29,
-     'adx-enabled': False,
-     'rsi-enabled': True,
-     'trigger': 'bb_lower'}
+    # Buy hyperspace params:
+    buy_params = {
+        'buy_adx': 44,
+        'buy_rsi': 29,
+        'buy_adx_enabled': False,
+        'buy_rsi_enabled': True,
+        'buy_trigger': 'bb_lower'
+    }
 ```
 
 You should understand this result like:
 
-- The buy trigger that worked best was `bb_lower`.
-- You should not use ADX because `adx-enabled: False`)
-- You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`)
+* The buy trigger that worked best was `bb_lower`.
+* You should not use ADX because `'buy_adx_enabled': False`.
+* You should **consider** using the RSI indicator (`'buy_rsi_enabled': True`) and the best value is `29.0` (`'buy_rsi': 29.0`)
 
-You have to look inside your strategy file into `buy_strategy_generator()`
-method, what those values match to.
+Your strategy class can immediately take advantage of these results. Simply copy hyperopt results block and paste them at class level, replacing old parameters (if any). New parameters will automatically be loaded next time strategy is executed.
 
-So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block:
+Transferring your whole hyperopt result to your strategy would then look like:
 
 ```python
-(dataframe['rsi'] < 29.0)
+class MyAwesomeStrategy(IStrategy):
+    # Buy hyperspace params:
+    buy_params = {
+        'buy_adx': 44,
+        'buy_rsi': 29,
+        'buy_adx_enabled': False,
+        'buy_rsi_enabled': True,
+        'buy_trigger': 'bb_lower'
+    }
 ```
 
-Translating your whole hyperopt result as the new buy-signal would then look like:
-
-```python
-def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame:
-    dataframe.loc[
-        (
-            (dataframe['rsi'] < 29.0) &  # rsi-value
-            dataframe['close'] < dataframe['bb_lowerband']  # trigger
-        ),
-        'buy'] = 1
-    return dataframe
-```
-
-By default, hyperopt prints colorized results -- epochs with positive profit are printed in the green color. This highlighting helps you find epochs that can be interesting for later analysis. Epochs with zero total profit or with negative profits (losses) are printed in the normal color. If you do not need colorization of results (for instance, when you are redirecting hyperopt output to a file) you can switch colorization off by specifying the `--no-color` option in the command line.
-
-You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option.
-
-!!! Note "Windows and color output"
-    Windows does not support color-output natively, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL.
-
 ### Understand Hyperopt ROI results
 
 If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'default' or 'roi'), your result will look as follows and include a ROI table:
@@ -499,11 +533,13 @@ Best result:
 
     44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
 
-ROI table:
-{   0: 0.10674,
-    21: 0.09158,
-    78: 0.03634,
-    118: 0}
+    # ROI table:
+    minimal_roi = {
+        0: 0.10674,
+        21: 0.09158,
+        78: 0.03634,
+        118: 0
+    }
 ```
 
 In order to use this best ROI table found by Hyperopt in backtesting and for live trades/dry-run, copy-paste it as the value of the `minimal_roi` attribute of your custom strategy:
@@ -523,23 +559,26 @@ As stated in the comment, you can also use it as the value of the `minimal_roi`
 
 #### Default ROI Search Space
 
-If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the timeframe used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 5 digits after the decimal point):
+If you are optimizing ROI, Freqtrade creates the 'roi' optimization hyperspace for you -- it's the hyperspace of components for the ROI tables. By default, each ROI table generated by the Freqtrade consists of 4 rows (steps). Hyperopt implements adaptive ranges for ROI tables with ranges for values in the ROI steps that depend on the timeframe used. By default the values vary in the following ranges (for some of the most used timeframes, values are rounded to 3 digits after the decimal point):
 
-| # step | 1m     |                   | 5m       |             | 1h         |                   | 1d           |                   |
-| ------ | ------ | ----------------- | -------- | ----------- | ---------- | ----------------- | ------------ | ----------------- |
-| 1      | 0      | 0.01161...0.11992 | 0        | 0.03...0.31 | 0          | 0.06883...0.71124 | 0            | 0.12178...1.25835 |
-| 2      | 2...8  | 0.00774...0.04255 | 10...40  | 0.02...0.11 | 120...480  | 0.04589...0.25238 | 2880...11520 | 0.08118...0.44651 |
-| 3      | 4...20 | 0.00387...0.01547 | 20...100 | 0.01...0.04 | 240...1200 | 0.02294...0.09177 | 5760...28800 | 0.04059...0.16237 |
-| 4      | 6...44 | 0.0               | 30...220 | 0.0         | 360...2640 | 0.0               | 8640...63360 | 0.0               |
+| # step | 1m     |               | 5m       |             | 1h         |               | 1d           |               |
+| ------ | ------ | ------------- | -------- | ----------- | ---------- | ------------- | ------------ | ------------- |
+| 1      | 0      | 0.011...0.119 | 0        | 0.03...0.31 | 0          | 0.068...0.711 | 0            | 0.121...1.258 |
+| 2      | 2...8  | 0.007...0.042 | 10...40  | 0.02...0.11 | 120...480  | 0.045...0.252 | 2880...11520 | 0.081...0.446 |
+| 3      | 4...20 | 0.003...0.015 | 20...100 | 0.01...0.04 | 240...1200 | 0.022...0.091 | 5760...28800 | 0.040...0.162 |
+| 4      | 6...44 | 0.0           | 30...220 | 0.0         | 360...2640 | 0.0           | 8640...63360 | 0.0           |
 
 These ranges should be sufficient in most cases. The minutes in the steps (ROI dict keys) are scaled linearly depending on the timeframe used. The ROI values in the steps (ROI dict values) are scaled logarithmically depending on the timeframe used.
 
 If you have the `generate_roi_table()` and `roi_space()` methods in your custom hyperopt file, remove them in order to utilize these adaptive ROI tables and the ROI hyperoptimization space generated by Freqtrade by default.
 
-Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). 
+Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps).
 
 A sample for these methods can be found in [sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py).
 
+!!! Note "Reduced search space"
+    To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs.
+
 ### Understand Hyperopt Stoploss results
 
 If you are optimizing stoploss values (i.e. if optimization search-space contains 'all', 'default' or 'stoploss'), your result will look as follows and include stoploss:
@@ -549,13 +588,16 @@ Best result:
 
     44/100:    135 trades. Avg profit  0.57%. Total profit  0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367
 
-Buy hyperspace params:
-{   'adx-value': 44,
-    'rsi-value': 29,
-    'adx-enabled': False,
-    'rsi-enabled': True,
-    'trigger': 'bb_lower'}
-Stoploss: -0.27996
+    # Buy hyperspace params:
+    buy_params = {
+        'buy_adx': 44,
+        'buy_rsi': 29,
+        'buy_adx_enabled': False,
+        'buy_rsi_enabled': True,
+        'buy_trigger': 'bb_lower'
+    }
+
+    stoploss: -0.27996
 ```
 
 In order to use this best stoploss value found by Hyperopt in backtesting and for live trades/dry-run, copy-paste it as the value of the `stoploss` attribute of your custom strategy:
@@ -576,6 +618,9 @@ If you have the `stoploss_space()` method in your custom hyperopt file, remove i
 
 Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py).
 
+!!! Note "Reduced search space"
+    To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs.
+
 ### Understand Hyperopt Trailing Stop results
 
 If you are optimizing trailing stop values (i.e. if optimization search-space contains 'all' or 'trailing'), your result will look as follows and include trailing stop parameters:
@@ -585,11 +630,11 @@ Best result:
 
     45/100:    606 trades. Avg profit  1.04%. Total profit  0.31555614 BTC ( 630.48Σ%). Avg duration 150.3 mins. Objective: -1.10161
 
-Trailing stop:
-{   'trailing_only_offset_is_reached': True,
-    'trailing_stop': True,
-    'trailing_stop_positive': 0.02001,
-    'trailing_stop_positive_offset': 0.06038}
+    # Trailing stop:
+    trailing_stop = True
+    trailing_stop_positive = 0.02001
+    trailing_stop_positive_offset = 0.06038
+    trailing_only_offset_is_reached = True
 ```
 
 In order to use these best trailing stop parameters found by Hyperopt in backtesting and for live trades/dry-run, copy-paste them as the values of the corresponding attributes of your custom strategy:
@@ -611,6 +656,59 @@ If you are optimizing trailing stop values, Freqtrade creates the 'trailing' opt
 
 Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py).
 
+!!! Note "Reduced search space"
+    To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs.
+
+### Reproducible results
+
+The search for optimal parameters starts with a few (currently 30) random combinations in the hyperspace of parameters, random Hyperopt epochs. These random epochs are marked with an asterisk character (`*`) in the first column in the Hyperopt output.
+
+The initial state for generation of these random values (random state) is controlled by the value of the `--random-state` command line option. You can set it to some arbitrary value of your choice to obtain reproducible results.
+
+If you have not set this value explicitly in the command line options, Hyperopt seeds the random state with some random value for you. The random state value for each Hyperopt run is shown in the log, so you can copy and paste it into the `--random-state` command line option to repeat the set of the initial random epochs used.
+
+If you have not changed anything in the command line options, configuration, timerange, Strategy and Hyperopt classes, historical data and the Loss Function -- you should obtain same hyper-optimization results with same random state value used.
+
+## Output formatting
+
+By default, hyperopt prints colorized results -- epochs with positive profit are printed in the green color. This highlighting helps you find epochs that can be interesting for later analysis. Epochs with zero total profit or with negative profits (losses) are printed in the normal color. If you do not need colorization of results (for instance, when you are redirecting hyperopt output to a file) you can switch colorization off by specifying the `--no-color` option in the command line.
+
+You can use the `--print-all` command line option if you would like to see all results in the hyperopt output, not only the best ones. When `--print-all` is used, current best results are also colorized by default -- they are printed in bold (bright) style. This can also be switched off with the `--no-color` command line option.
+
+!!! Note "Windows and color output"
+    Windows does not support color-output natively, therefore it is automatically disabled. To have color-output for hyperopt running under windows, please consider using WSL.
+
+## Position stacking and disabling max market positions
+
+In some situations, you may need to run Hyperopt (and Backtesting) with the
+`--eps`/`--enable-position-staking` and `--dmmp`/`--disable-max-market-positions` arguments.
+
+By default, hyperopt emulates the behavior of the Freqtrade Live Run/Dry Run, where only one
+open trade is allowed for every traded pair. The total number of trades open for all pairs
+is also limited by the `max_open_trades` setting. During Hyperopt/Backtesting this may lead to
+some potential trades to be hidden (or masked) by previously open trades.
+
+The `--eps`/`--enable-position-stacking` argument allows emulation of buying the same pair multiple times,
+while `--dmmp`/`--disable-max-market-positions` disables applying `max_open_trades`
+during Hyperopt/Backtesting (which is equal to setting `max_open_trades` to a very high
+number).
+
+!!! Note
+    Dry/live runs will **NOT** use position stacking - therefore it does make sense to also validate the strategy without this as it's closer to reality.
+
+You can also enable position stacking in the configuration file by explicitly setting
+`"position_stacking"=true`.
+
+## Out of Memory errors
+
+As hyperopt consumes a lot of memory (the complete data needs to be in memory once per parallel backtesting process), it's likely that you run into "out of memory" errors.
+To combat these, you have multiple options:
+
+* reduce the amount of pairs
+* reduce the timerange used (`--timerange <timerange>`)
+* reduce the number of parallel processes (`-j <n>`)
+* Increase the memory of your machine
+
 ## Show details of Hyperopt results
 
 After you run Hyperopt for the desired amount of epochs, you can later list all results for analysis, select only best or profitable once, and show the details for any of the epochs previously evaluated. This can be done with the `hyperopt-list` and `hyperopt-show` sub-commands. The usage of these sub-commands is described in the [Utils](utils.md#list-hyperopt-results) chapter.

docs/includes/pairlists.md

@@ -4,7 +4,7 @@ Pairlist Handlers define the list of pairs (pairlist) that the bot should trade.
 
 In your configuration, you can use Static Pairlist (defined by the [`StaticPairList`](#static-pair-list) Pairlist Handler) and Dynamic Pairlist (defined by the [`VolumePairList`](#volume-pair-list) Pairlist Handler).
 
-Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter) and [`SpreadFilter`](#spreadfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
+Additionally, [`AgeFilter`](#agefilter), [`PrecisionFilter`](#precisionfilter), [`PriceFilter`](#pricefilter), [`ShuffleFilter`](#shufflefilter), [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) act as Pairlist Filters, removing certain pairs and/or moving their positions in the pairlist.
 
 If multiple Pairlist Handlers are used, they are chained and a combination of all Pairlist Handlers forms the resulting pairlist the bot uses for trading and backtesting. Pairlist Handlers are executed in the sequence they are configured. You should always configure either `StaticPairList` or `VolumePairList` as the starting Pairlist Handler.
 
@@ -29,6 +29,7 @@ You may also use something like `.*DOWN/BTC` or `.*UP/BTC` to exclude leveraged
 * [`ShuffleFilter`](#shufflefilter)
 * [`SpreadFilter`](#spreadfilter)
 * [`RangeStabilityFilter`](#rangestabilityfilter)
+* [`VolatilityFilter`](#volatilityfilter)
 
 !!! Tip "Testing pairlists"
     Pairlist configurations can be quite tricky to get right. Best use the [`test-pairlist`](utils.md#test-pairlist) utility sub-command to test your configuration quickly.
@@ -59,6 +60,8 @@ When used in the chain of Pairlist Handlers in a non-leading position (after Sta
 When used on the leading position of the chain of Pairlist Handlers, it does not consider `pair_whitelist` configuration setting, but selects the top assets from all available markets (with matching stake-currency) on the exchange.
 
 The `refresh_period` setting allows to define the period (in seconds), at which the pairlist will be refreshed. Defaults to 1800s (30 minutes).
+The pairlist cache (`refresh_period`) on `VolumePairList` is only applicable to generating pairlists.
+Filtering instances (not the first position in the list) will not apply any cache and will always use up-to-date data.
 
 `VolumePairList` is based on the ticker data from exchange, as reported by the ccxt library:
 
@@ -89,6 +92,7 @@ This filter allows freqtrade to ignore pairs until they have been listed for at
 #### PerformanceFilter
 
 Sorts pairs by past trade performance, as follows:
+
 1. Positive performance.
 2. No closed trades yet.
 3. Negative performance.
@@ -164,9 +168,32 @@ If the trading range over the last 10 days is <1%, remove the pair from the whit
 !!! Tip
     This Filter can be used to automatically remove stable coin pairs, which have a very low trading range, and are therefore extremely difficult to trade with profit.
 
+#### VolatilityFilter
+
+Volatility is the degree of historical variation of a pairs over time, is is measured by the standard deviation of logarithmic daily returns. Returns are assumed to be normally distributed, although actual distribution might be different. In a normal distribution, 68% of observations fall within one standard deviation and 95% of observations fall within two standard deviations. Assuming a volatility of 0.05 means that the expected returns for 20 out of 30 days is expected to be less than 5% (one standard deviation). Volatility is a positive ratio of the expected deviation of return and can be greater than 1.00. Please refer to the wikipedia definition of [`volatility`](https://en.wikipedia.org/wiki/Volatility_(finance)).
+
+This filter removes pairs if the average volatility over a `lookback_days` days is below `min_volatility` or above `max_volatility`. Since this is a filter that requires additional data, the results are cached for `refresh_period`.
+
+This filter can be used to narrow down your pairs to a certain volatility or avoid very volatile pairs. 
+
+In the below example:
+If the volatility over the last 10 days is not in the range of 0.05-0.50, remove the pair from the whitelist. The filter is applied every 24h.
+
+```json
+"pairlists": [
+    {
+        "method": "VolatilityFilter",
+        "lookback_days": 10,
+        "min_volatility": 0.05,
+        "max_volatility": 0.50,
+        "refresh_period": 86400
+    }
+]
+```
+
 ### Full example of Pairlist Handlers
 
-The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies both [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the `SpreadFilter` is applied and pairs are finally shuffled with the random seed set to some predefined value.
+The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets, sorting pairs by `quoteVolume` and applies [`PrecisionFilter`](#precisionfilter) and [`PriceFilter`](#price-filter), filtering all assets where 1 price unit is > 1%. Then the [`SpreadFilter`](#spreadfilter) and [`VolatilityFilter`](#volatilityfilter) is applied and pairs are finally shuffled with the random seed set to some predefined value.
 
 ```json
 "exchange": {
@@ -189,6 +216,13 @@ The below example blacklists `BNB/BTC`, uses `VolumePairList` with `20` assets,
         "min_rate_of_change": 0.01,
         "refresh_period": 1440
     },
+    {
+        "method": "VolatilityFilter",
+        "lookback_days": 10,
+        "min_volatility": 0.05,
+        "max_volatility": 0.50,
+        "refresh_period": 86400
+    },
     {"method": "ShuffleFilter", "seed": 42}
     ],
 ```

docs/index.md

@@ -1,4 +1,5 @@
-# Freqtrade
+![freqtrade](assets/freqtrade_poweredby.svg)
+
 [![Freqtrade CI](https://github.com/freqtrade/freqtrade/workflows/Freqtrade%20CI/badge.svg)](https://github.com/freqtrade/freqtrade/actions/)
 [![Coverage Status](https://coveralls.io/repos/github/freqtrade/freqtrade/badge.svg?branch=develop&service=github)](https://coveralls.io/github/freqtrade/freqtrade?branch=develop)
 [![Maintainability](https://api.codeclimate.com/v1/badges/5737e6d668200b7518ff/maintainability)](https://codeclimate.com/github/freqtrade/freqtrade/maintainability)
@@ -39,7 +40,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual,
 - [X] [Bittrex](https://bittrex.com/)
 - [X] [FTX](https://ftx.com)
 - [X] [Kraken](https://kraken.com/)
-- [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
+- [ ] [potentially many others through <img alt="ccxt" width="30px" src="assets/ccxt-logo.svg" />](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_
 
 ### Community tested
 

docs/plotting.md

@@ -37,7 +37,7 @@ usage: freqtrade plot-dataframe [-h] [-v] [--logfile FILE] [-V] [-c PATH]
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
-                        Show profits for only these pairs. Pairs are space-
+                        Limit command to these pairs. Pairs are space-
                         separated.
   --indicators1 INDICATORS1 [INDICATORS1 ...]
                         Set indicators from your strategy you want in the
@@ -66,8 +66,7 @@ optional arguments:
   --timerange TIMERANGE
                         Specify what timerange of data to use.
   -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
-                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
-                        `1d`).
+                        Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
   --no-trades           Skip using trades from backtesting file and DB.
 
 Common arguments:
@@ -91,6 +90,7 @@ Strategy arguments:
                         Specify strategy class name which will be used by the
                         bot.
   --strategy-path PATH  Specify additional strategy lookup path.
+
 ```
 
 Example:
@@ -245,7 +245,7 @@ usage: freqtrade plot-profit [-h] [-v] [--logfile FILE] [-V] [-c PATH]
 optional arguments:
   -h, --help            show this help message and exit
   -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...]
-                        Show profits for only these pairs. Pairs are space-
+                        Limit command to these pairs. Pairs are space-
                         separated.
   --timerange TIMERANGE
                         Specify what timerange of data to use.
@@ -264,8 +264,7 @@ optional arguments:
                         Specify the source for trades (Can be DB or file
                         (backtest file)) Default: file
   -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME
-                        Specify ticker interval (`1m`, `5m`, `30m`, `1h`,
-                        `1d`).
+                        Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).
 
 Common arguments:
   -v, --verbose         Verbose mode (-vv for more, -vvv to get all messages).
@@ -288,6 +287,7 @@ Strategy arguments:
                         Specify strategy class name which will be used by the
                         bot.
   --strategy-path PATH  Specify additional strategy lookup path.
+
 ```
 
 The `-p/--pairs`  argument, can be used to limit the pairs that are considered for this calculation.

docs/requirements-docs.txt

@@ -1,3 +1,3 @@
-mkdocs-material==7.0.6
+mkdocs-material==7.1.3
 mdx_truly_sane_lists==1.2
 pymdown-extensions==8.1.1

docs/rest-api.md

@@ -124,7 +124,8 @@ python3 scripts/rest_client.py --config rest_config.json <command> [optional par
 | `stop` | Stops the trader.
 | `stopbuy` | Stops the trader from opening new trades. Gracefully closes open trades according to their rules.
 | `reload_config` | Reloads the configuration file.
-| `trades` | List last trades.
+| `trades` | List last trades. Limited to 500 trades per call.
+| `trade/<tradeid>` | Get specific trade.
 | `delete_trade <trade_id>` | Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange.
 | `show_config` | Shows part of the current configuration with relevant settings to operation.
 | `logs` | Shows last log messages.
@@ -181,7 +182,7 @@ count
 	Return the amount of open trades.
 
 daily
-	Return the amount of open trades.
+	Return the profits for each day, and amount of trades.
 
 delete_lock
 	Delete (disable) lock from the database.
@@ -214,7 +215,7 @@ locks
 logs
 	Show latest logs.
 
-        :param limit: Limits log messages to the last <limit> logs. No limit to get all the trades.
+        :param limit: Limits log messages to the last <limit> logs. No limit to get the entire log.
 
 pair_candles
 	Return live dataframe for <pair><timeframe>.
@@ -234,6 +235,9 @@ pair_history
 performance
 	Return the performance of the different coins.
 
+ping
+	simple ping
+
 plot_config
 	Return plot configuration if the strategy defines one.
 
@@ -270,17 +274,22 @@ strategy
 
         :param strategy: Strategy class name
 
-trades
-	Return trades history.
+trade
+	Return specific trade
 
-        :param limit: Limits trades to the X last trades. No limit to get all the trades.
+        :param trade_id: Specify which trade to get.
+
+trades
+	Return trades history, sorted by id
+
+        :param limit: Limits trades to the X last trades. Max 500 trades.
+        :param offset: Offset by this amount of trades.
 
 version
 	Return the version of the bot.
 
 whitelist
 	Show the current whitelist.
-
 ```
 
 ### OpenAPI interface

docs/strategy-advanced.md

@@ -57,7 +57,7 @@ class AwesomeStrategy(IStrategy):
         dataframe['atr'] = ta.ATR(dataframe)
         if self.dp.runmode.value in ('backtest', 'hyperopt'):
           # add indicator mapped to correct DatetimeIndex to custom_info
-          self.custom_info[metadata['pair']] = dataframe[['date', 'atr']].copy().set_index('date')
+          self.custom_info[metadata['pair']] = dataframe[['date', 'atr']].set_index('date')
         return dataframe
 ```
 

docs/strategy_analysis_example.md

@@ -195,4 +195,18 @@ graph.show(renderer="browser")
 
 ```
 
+## Plot average profit per trade as distribution graph
+
+
+```python
+import plotly.figure_factory as ff
+
+hist_data = [trades.profit_ratio]
+group_labels = ['profit_ratio']  # name of the dataset
+
+fig = ff.create_distplot(hist_data, group_labels,bin_size=0.01)
+fig.show()
+
+```
+
 Feel free to submit an issue or Pull Request enhancing this document if you would like to share ideas on how to best analyze the data.

docs/telegram-usage.md

@@ -82,12 +82,19 @@ Example configuration showing the different settings:
          "buy": "silent",
          "sell": "on",
          "buy_cancel": "silent",
-         "sell_cancel": "on"
+         "sell_cancel": "on",
+         "buy_fill": "off",
+         "sell_fill": "off"
       },
       "balance_dust_level": 0.01
    },
 ```
 
+`buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange.
+`sell` notifications are sent when the order is placed, while `sell_fill` notifications are sent when the order is filled on the exchange.
+`*_fill` notifications are off by default and must be explicitly enabled.
+
+
 `balance_dust_level` will define what the `/balance` command takes as "dust" - Currencies with a balance below this will be shown.
 
 ## Create a custom keyboard (command shortcut buttons)

docs/utils.md

@@ -253,18 +253,211 @@ optional arguments:
 * Example: see exchanges available for the bot:
 ```
 $ freqtrade list-exchanges
-Exchanges available for Freqtrade: _1btcxe, acx, allcoin, bequant, bibox, binance, binanceje, binanceus, bitbank, bitfinex, bitfinex2, bitkk, bitlish, bitmart, bittrex, bitz, bleutrade, btcalpha, btcmarkets, btcturk, buda, cex, cobinhood, coinbaseprime, coinbasepro, coinex, cointiger, coss, crex24, digifinex, dsx, dx, ethfinex, fcoin, fcoinjp, gateio, gdax, gemini, hitbtc2, huobipro, huobiru, idex, kkex, kraken, kucoin, kucoin2, kuna, lbank, mandala, mercado, oceanex, okcoincny, okcoinusd, okex, okex3, poloniex, rightbtc, theocean, tidebit, upbit, zb
+Exchanges available for Freqtrade:
+Exchange name    Valid    reason
+---------------  -------  --------------------------------------------
+aax              True
+ascendex         True     missing opt: fetchMyTrades
+bequant          True
+bibox            True
+bigone           True
+binance          True
+binanceus        True
+bitbank          True     missing opt: fetchTickers
+bitcoincom       True
+bitfinex         True
+bitforex         True     missing opt: fetchMyTrades, fetchTickers
+bitget           True
+bithumb          True     missing opt: fetchMyTrades
+bitkk            True     missing opt: fetchMyTrades
+bitmart          True
+bitmax           True     missing opt: fetchMyTrades
+bitpanda         True
+bittrex          True
+bitvavo          True
+bitz             True     missing opt: fetchMyTrades
+btcalpha         True     missing opt: fetchTicker, fetchTickers
+btcmarkets       True     missing opt: fetchTickers
+buda             True     missing opt: fetchMyTrades, fetchTickers
+bw               True     missing opt: fetchMyTrades, fetchL2OrderBook
+bybit            True
+bytetrade        True
+cdax             True
+cex              True     missing opt: fetchMyTrades
+coinbaseprime    True     missing opt: fetchTickers
+coinbasepro      True     missing opt: fetchTickers
+coinex           True
+crex24           True
+deribit          True
+digifinex        True
+equos            True     missing opt: fetchTicker, fetchTickers
+eterbase         True
+fcoin            True     missing opt: fetchMyTrades, fetchTickers
+fcoinjp          True     missing opt: fetchMyTrades, fetchTickers
+ftx              True
+gateio           True
+gemini           True
+gopax            True
+hbtc             True
+hitbtc           True
+huobijp          True
+huobipro         True
+idex             True
+kraken           True
+kucoin           True
+lbank            True     missing opt: fetchMyTrades
+mercado          True     missing opt: fetchTickers
+ndax             True     missing opt: fetchTickers
+novadax          True
+okcoin           True
+okex             True
+probit           True
+qtrade           True
+stex             True
+timex            True
+upbit            True     missing opt: fetchMyTrades
+vcc              True
+zb               True     missing opt: fetchMyTrades
+
 ```
 
+!!! Note "missing opt exchanges"
+    Values with "missing opt:" might need special configuration (e.g. using orderbook if `fetchTickers` is missing) - but should in theory work (although we cannot guarantee they will).
+
 * Example: see all exchanges supported by the ccxt library (including 'bad' ones, i.e. those that are known to not work with Freqtrade):
 ```
 $ freqtrade list-exchanges -a
-All exchanges supported by the ccxt library: _1btcxe, acx, adara, allcoin, anxpro, bcex, bequant, bibox, bigone, binance, binanceje, binanceus, bit2c, bitbank, bitbay, bitfinex, bitfinex2, bitflyer, bitforex, bithumb, bitkk, bitlish, bitmart, bitmex, bitso, bitstamp, bitstamp1, bittrex, bitz, bl3p, bleutrade, braziliex, btcalpha, btcbox, btcchina, btcmarkets, btctradeim, btctradeua, btcturk, buda, bxinth, cex, chilebit, cobinhood, coinbase, coinbaseprime, coinbasepro, coincheck, coinegg, coinex, coinexchange, coinfalcon, coinfloor, coingi, coinmarketcap, coinmate, coinone, coinspot, cointiger, coolcoin, coss, crex24, crypton, deribit, digifinex, dsx, dx, ethfinex, exmo, exx, fcoin, fcoinjp, flowbtc, foxbit, fybse, gateio, gdax, gemini, hitbtc, hitbtc2, huobipro, huobiru, ice3x, idex, independentreserve, indodax, itbit, kkex, kraken, kucoin, kucoin2, kuna, lakebtc, latoken, lbank, liquid, livecoin, luno, lykke, mandala, mercado, mixcoins, negociecoins, nova, oceanex, okcoincny, okcoinusd, okex, okex3, paymium, poloniex, rightbtc, southxchange, stronghold, surbitcoin, theocean, therock, tidebit, tidex, upbit, vaultoro, vbtc, virwox, xbtce, yobit, zaif, zb
+All exchanges supported by the ccxt library:
+Exchange name       Valid    reason
+------------------  -------  ---------------------------------------------------------------------------------------
+aax                 True
+aofex               False    missing: fetchOrder
+ascendex            True     missing opt: fetchMyTrades
+bequant             True
+bibox               True
+bigone              True
+binance             True
+binanceus           True
+bit2c               False    missing: fetchOrder, fetchOHLCV
+bitbank             True     missing opt: fetchTickers
+bitbay              False    missing: fetchOrder
+bitcoincom          True
+bitfinex            True
+bitfinex2           False    missing: fetchOrder
+bitflyer            False    missing: fetchOrder, fetchOHLCV
+bitforex            True     missing opt: fetchMyTrades, fetchTickers
+bitget              True
+bithumb             True     missing opt: fetchMyTrades
+bitkk               True     missing opt: fetchMyTrades
+bitmart             True
+bitmax              True     missing opt: fetchMyTrades
+bitmex              False    Various reasons.
+bitpanda            True
+bitso               False    missing: fetchOHLCV
+bitstamp            False    Does not provide history. Details in https://github.com/freqtrade/freqtrade/issues/1983
+bitstamp1           False    missing: fetchOrder, fetchOHLCV
+bittrex             True
+bitvavo             True
+bitz                True     missing opt: fetchMyTrades
+bl3p                False    missing: fetchOrder, fetchOHLCV
+bleutrade           False    missing: fetchOrder
+braziliex           False    missing: fetchOHLCV
+btcalpha            True     missing opt: fetchTicker, fetchTickers
+btcbox              False    missing: fetchOHLCV
+btcmarkets          True     missing opt: fetchTickers
+btctradeua          False    missing: fetchOrder, fetchOHLCV
+btcturk             False    missing: fetchOrder
+buda                True     missing opt: fetchMyTrades, fetchTickers
+bw                  True     missing opt: fetchMyTrades, fetchL2OrderBook
+bybit               True
+bytetrade           True
+cdax                True
+cex                 True     missing opt: fetchMyTrades
+chilebit            False    missing: fetchOrder, fetchOHLCV
+coinbase            False    missing: fetchOrder, cancelOrder, createOrder, fetchOHLCV
+coinbaseprime       True     missing opt: fetchTickers
+coinbasepro         True     missing opt: fetchTickers
+coincheck           False    missing: fetchOrder, fetchOHLCV
+coinegg             False    missing: fetchOHLCV
+coinex              True
+coinfalcon          False    missing: fetchOHLCV
+coinfloor           False    missing: fetchOrder, fetchOHLCV
+coingi              False    missing: fetchOrder, fetchOHLCV
+coinmarketcap       False    missing: fetchOrder, cancelOrder, createOrder, fetchBalance, fetchOHLCV
+coinmate            False    missing: fetchOHLCV
+coinone             False    missing: fetchOHLCV
+coinspot            False    missing: fetchOrder, cancelOrder, fetchOHLCV
+crex24              True
+currencycom         False    missing: fetchOrder
+delta               False    missing: fetchOrder
+deribit             True
+digifinex           True
+equos               True     missing opt: fetchTicker, fetchTickers
+eterbase            True
+exmo                False    missing: fetchOrder
+exx                 False    missing: fetchOHLCV
+fcoin               True     missing opt: fetchMyTrades, fetchTickers
+fcoinjp             True     missing opt: fetchMyTrades, fetchTickers
+flowbtc             False    missing: fetchOrder, fetchOHLCV
+foxbit              False    missing: fetchOrder, fetchOHLCV
+ftx                 True
+gateio              True
+gemini              True
+gopax               True
+hbtc                True
+hitbtc              True
+hollaex             False    missing: fetchOrder
+huobijp             True
+huobipro            True
+idex                True
+independentreserve  False    missing: fetchOHLCV
+indodax             False    missing: fetchOHLCV
+itbit               False    missing: fetchOHLCV
+kraken              True
+kucoin              True
+kuna                False    missing: fetchOHLCV
+lakebtc             False    missing: fetchOrder, fetchOHLCV
+latoken             False    missing: fetchOrder, fetchOHLCV
+lbank               True     missing opt: fetchMyTrades
+liquid              False    missing: fetchOHLCV
+luno                False    missing: fetchOHLCV
+lykke               False    missing: fetchOHLCV
+mercado             True     missing opt: fetchTickers
+mixcoins            False    missing: fetchOrder, fetchOHLCV
+ndax                True     missing opt: fetchTickers
+novadax             True
+oceanex             False    missing: fetchOHLCV
+okcoin              True
+okex                True
+paymium             False    missing: fetchOrder, fetchOHLCV
+phemex              False    Does not provide history.
+poloniex            False    missing: fetchOrder
+probit              True
+qtrade              True
+rightbtc            False    missing: fetchOrder
+ripio               False    missing: fetchOHLCV
+southxchange        False    missing: fetchOrder, fetchOHLCV
+stex                True
+surbitcoin          False    missing: fetchOrder, fetchOHLCV
+therock             False    missing: fetchOHLCV
+tidebit             False    missing: fetchOrder
+tidex               False    missing: fetchOHLCV
+timex               True
+upbit               True     missing opt: fetchMyTrades
+vbtc                False    missing: fetchOrder, fetchOHLCV
+vcc                 True
+wavesexchange       False    missing: fetchOrder
+whitebit            False    missing: fetchOrder, cancelOrder, createOrder, fetchBalance
+xbtce               False    missing: fetchOrder, fetchOHLCV
+xena                False    missing: fetchOrder
+yobit               False    missing: fetchOHLCV
+zaif                False    missing: fetchOrder, fetchOHLCV
+zb                  True     missing opt: fetchMyTrades
 ```
 
 ## List Timeframes
 
-Use the `list-timeframes` subcommand to see the list of timeframes (ticker intervals) available for the exchange.
+Use the `list-timeframes` subcommand to see the list of timeframes available for the exchange.
 
 ```
 usage: freqtrade list-timeframes [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [-1]

docs/webhook-config.md

@@ -19,6 +19,11 @@ Sample configuration (tested using IFTTT).
             "value1": "Cancelling Open Buy Order for {pair}",
             "value2": "limit {limit:8f}",
             "value3": "{stake_amount:8f} {stake_currency}"
+        },
+         "webhookbuyfill": {
+            "value1": "Buy Order for {pair} filled",
+            "value2": "at {open_rate:8f}",
+            "value3": ""
         },
         "webhooksell": {
             "value1": "Selling {pair}",
@@ -30,6 +35,11 @@ Sample configuration (tested using IFTTT).
             "value2": "limit {limit:8f}",
             "value3": "profit: {profit_amount:8f} {stake_currency} ({profit_ratio})"
         },
+        "webhooksellfill": {
+            "value1": "Sell Order for {pair} filled",
+            "value2": "at {close_rate:8f}.",
+            "value3": ""
+        },
         "webhookstatus": {
             "value1": "Status: {status}",
             "value2": "",
@@ -91,6 +101,21 @@ Possible parameters are:
 * `order_type`
 * `current_rate`
 
+### Webhookbuyfill
+
+The fields in `webhook.webhookbuyfill` are filled when the bot filled a buy order. Parameters are filled using string.format.
+Possible parameters are:
+
+* `trade_id`
+* `exchange`
+* `pair`
+* `open_rate`
+* `amount`
+* `open_date`
+* `stake_amount`
+* `stake_currency`
+* `fiat_currency`
+
 ### Webhooksell
 
 The fields in `webhook.webhooksell` are filled when the bot sells a trade. Parameters are filled using string.format.
@@ -103,6 +128,27 @@ Possible parameters are:
 * `limit`
 * `amount`
 * `open_rate`
+* `profit_amount`
+* `profit_ratio`
+* `stake_currency`
+* `fiat_currency`
+* `sell_reason`
+* `order_type`
+* `open_date`
+* `close_date`
+
+### Webhooksellfill
+
+The fields in `webhook.webhooksellfill` are filled when the bot fills a sell order (closes a Trae). Parameters are filled using string.format.
+Possible parameters are:
+
+* `trade_id`
+* `exchange`
+* `pair`
+* `gain`
+* `close_rate`
+* `amount`
+* `open_rate`
 * `current_rate`
 * `profit_amount`
 * `profit_ratio`

environment.yml

@@ -4,7 +4,7 @@ channels:
 # - defaults
 dependencies:
 # 1/4 req main
-    - python>=3.7
+    - python>=3.7,<3.9
     - numpy
     - pandas
     - pip

freqtrade/__init__.py

@@ -1,5 +1,5 @@
 """ Freqtrade bot """
-__version__ = '2021.3'
+__version__ = '2021.4'
 
 if __version__ == 'develop':
 

freqtrade/commands/arguments.py

@@ -17,7 +17,7 @@ ARGS_STRATEGY = ["strategy", "strategy_path"]
 ARGS_TRADE = ["db_url", "sd_notify", "dry_run", "dry_run_wallet", "fee"]
 
 ARGS_COMMON_OPTIMIZE = ["timeframe", "timerange", "dataformat_ohlcv",
-                        "max_open_trades", "stake_amount", "fee"]
+                        "max_open_trades", "stake_amount", "fee", "pairs"]
 
 ARGS_BACKTEST = ARGS_COMMON_OPTIMIZE + ["position_stacking", "use_max_market_positions",
                                         "enable_protections", "dry_run_wallet",
@@ -60,8 +60,9 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"]
 
 ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"]
 
-ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "timerange", "download_trades", "exchange",
-                      "timeframes", "erase", "dataformat_ohlcv", "dataformat_trades"]
+ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "timerange",
+                      "download_trades", "exchange", "timeframes", "erase", "dataformat_ohlcv",
+                      "dataformat_trades"]
 
 ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit",
                        "db_url", "trade_source", "export", "exportfilename",

freqtrade/commands/build_config_commands.py

@@ -1,9 +1,11 @@
 import logging
+import secrets
 from pathlib import Path
 from typing import Any, Dict, List
 
 from questionary import Separator, prompt
 
+from freqtrade.configuration.directory_operations import chown_user_directory
 from freqtrade.constants import UNLIMITED_STAKE_AMOUNT
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges
@@ -138,6 +140,32 @@ def ask_user_config() -> Dict[str, Any]:
             "message": "Insert Telegram chat id",
             "when": lambda x: x['telegram']
         },
+        {
+            "type": "confirm",
+            "name": "api_server",
+            "message": "Do you want to enable the Rest API (includes FreqUI)?",
+            "default": False,
+        },
+        {
+            "type": "text",
+            "name": "api_server_listen_addr",
+            "message": "Insert Api server Listen Address (best left untouched default!)",
+            "default": "127.0.0.1",
+            "when": lambda x: x['api_server']
+        },
+        {
+            "type": "text",
+            "name": "api_server_username",
+            "message": "Insert api-server username",
+            "default": "freqtrader",
+            "when": lambda x: x['api_server']
+        },
+        {
+            "type": "text",
+            "name": "api_server_password",
+            "message": "Insert api-server password",
+            "when": lambda x: x['api_server']
+        },
     ]
     answers = prompt(questions)
 
@@ -145,6 +173,9 @@ def ask_user_config() -> Dict[str, Any]:
         # Interrupted questionary sessions return an empty dict.
         raise OperationalException("User interrupted interactive questions.")
 
+    # Force JWT token to be a random string
+    answers['api_server_jwt_key'] = secrets.token_hex()
+
     return answers
 
 
@@ -186,6 +217,7 @@ def start_new_config(args: Dict[str, Any]) -> None:
     """
 
     config_path = Path(args['config'][0])
+    chown_user_directory(config_path.parent)
     if config_path.exists():
         overwrite = ask_user_overwrite(config_path)
         if overwrite:

freqtrade/commands/cli_options.py

@@ -118,7 +118,7 @@ AVAILABLE_CLI_OPTIONS = {
     # Optimize common
     "timeframe": Arg(
         '-i', '--timeframe', '--ticker-interval',
-        help='Specify ticker interval (`1m`, `5m`, `30m`, `1h`, `1d`).',
+        help='Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`).',
     ),
     "timerange": Arg(
         '--timerange',
@@ -195,6 +195,7 @@ AVAILABLE_CLI_OPTIONS = {
         '--hyperopt',
         help='Specify hyperopt class name which will be used by the bot.',
         metavar='NAME',
+        required=False,
     ),
     "hyperopt_path": Arg(
         '--hyperopt-path',
@@ -266,7 +267,7 @@ AVAILABLE_CLI_OPTIONS = {
         default=1,
     ),
     "hyperopt_loss": Arg(
-        '--hyperopt-loss',
+        '--hyperopt-loss', '--hyperoptloss',
         help='Specify the class name of the hyperopt loss function class (IHyperOptLoss). '
         'Different functions can generate completely different results, '
         'since the target for optimization is different. Built-in Hyperopt-loss-functions are: '
@@ -329,7 +330,7 @@ AVAILABLE_CLI_OPTIONS = {
     # Script options
     "pairs": Arg(
         '-p', '--pairs',
-        help='Show profits for only these pairs. Pairs are space-separated.',
+        help='Limit command to these pairs. Pairs are space-separated.',
         nargs='+',
     ),
     # Download data
@@ -344,6 +345,12 @@ AVAILABLE_CLI_OPTIONS = {
         type=check_int_positive,
         metavar='INT',
     ),
+    "new_pairs_days": Arg(
+        '--new-pairs-days',
+        help='Download data of new pairs for given number of days. Default: `%(default)s`.',
+        type=check_int_positive,
+        metavar='INT',
+    ),
     "download_trades": Arg(
         '--dl-trades',
         help='Download trades instead of OHLCV data. The bot will resample trades to the '

freqtrade/commands/data_commands.py

@@ -62,8 +62,8 @@ def start_download_data(args: Dict[str, Any]) -> None:
         if config.get('download_trades'):
             pairs_not_available = refresh_backtest_trades_data(
                 exchange, pairs=expanded_pairs, datadir=config['datadir'],
-                timerange=timerange, erase=bool(config.get('erase')),
-                data_format=config['dataformat_trades'])
+                timerange=timerange, new_pairs_days=config['new_pairs_days'],
+                erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
 
             # Convert downloaded trade data to different timeframes
             convert_trades_to_ohlcv(
@@ -75,8 +75,9 @@ def start_download_data(args: Dict[str, Any]) -> None:
         else:
             pairs_not_available = refresh_backtest_ohlcv_data(
                 exchange, pairs=expanded_pairs, timeframes=config['timeframes'],
-                datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
-                data_format=config['dataformat_ohlcv'])
+                datadir=config['datadir'], timerange=timerange,
+                new_pairs_days=config['new_pairs_days'],
+                erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'])
 
     except KeyboardInterrupt:
         sys.exit("SIGINT received, aborting ...")

freqtrade/commands/list_commands.py

@@ -13,7 +13,7 @@ from tabulate import tabulate
 from freqtrade.configuration import setup_utils_configuration
 from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES
 from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import available_exchanges, ccxt_exchanges, market_is_active
+from freqtrade.exchange import market_is_active, validate_exchanges
 from freqtrade.misc import plural
 from freqtrade.resolvers import ExchangeResolver, StrategyResolver
 from freqtrade.state import RunMode
@@ -28,14 +28,18 @@ def start_list_exchanges(args: Dict[str, Any]) -> None:
     :param args: Cli args from Arguments()
     :return: None
     """
-    exchanges = ccxt_exchanges() if args['list_exchanges_all'] else available_exchanges()
+    exchanges = validate_exchanges(args['list_exchanges_all'])
+
     if args['print_one_column']:
-        print('\n'.join(exchanges))
+        print('\n'.join([e[0] for e in exchanges]))
     else:
         if args['list_exchanges_all']:
-            print(f"All exchanges supported by the ccxt library: {', '.join(exchanges)}")
+            print("All exchanges supported by the ccxt library:")
         else:
-            print(f"Exchanges available for Freqtrade: {', '.join(exchanges)}")
+            print("Exchanges available for Freqtrade:")
+            exchanges = [e for e in exchanges if e[1] is not False]
+
+        print(tabulate(exchanges, headers=['Exchange name', 'Valid', 'reason']))
 
 
 def _print_objs_tabular(objs: List, print_colorized: bool) -> None:
@@ -99,7 +103,7 @@ def start_list_hyperopts(args: Dict[str, Any]) -> None:
 
 def start_list_timeframes(args: Dict[str, Any]) -> None:
     """
-    Print ticker intervals (timeframes) available on Exchange
+    Print timeframes available on Exchange
     """
     config = setup_utils_configuration(args, RunMode.UTIL_EXCHANGE)
     # Do not use timeframe set in the config
@@ -177,7 +181,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None:
             # human-readable formats.
             print()
 
-        if len(pairs):
+        if pairs:
             if args.get('print_list', False):
                 # print data as a list, with human-readable summary
                 print(f"{summary_str}: {', '.join(pairs.keys())}.")

freqtrade/configuration/check_exchange.py

@@ -2,8 +2,8 @@ import logging
 from typing import Any, Dict
 
 from freqtrade.exceptions import OperationalException
-from freqtrade.exchange import (available_exchanges, get_exchange_bad_reason, is_exchange_bad,
-                                is_exchange_known_ccxt, is_exchange_officially_supported)
+from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt,
+                                is_exchange_officially_supported, validate_exchange)
 from freqtrade.state import RunMode
 
 
@@ -57,9 +57,13 @@ def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool:
                 f'{", ".join(available_exchanges())}'
         )
 
-    if check_for_bad and is_exchange_bad(exchange):
-        raise OperationalException(f'Exchange "{exchange}" is known to not work with the bot yet. '
-                                   f'Reason: {get_exchange_bad_reason(exchange)}')
+    valid, reason = validate_exchange(exchange)
+    if not valid:
+        if check_for_bad:
+            raise OperationalException(f'Exchange "{exchange}"  will not work with Freqtrade. '
+                                       f'Reason: {reason}')
+        else:
+            logger.warning(f'Exchange "{exchange}"  will not work with Freqtrade. Reason: {reason}')
 
     if is_exchange_officially_supported(exchange):
         logger.info(f'Exchange "{exchange}" is officially supported '

freqtrade/configuration/config_validation.py

@@ -149,11 +149,6 @@ def _validate_edge(conf: Dict[str, Any]) -> None:
     if not conf.get('edge', {}).get('enabled'):
         return
 
-    if conf.get('pairlist', {}).get('method') == 'VolumePairList':
-        raise OperationalException(
-            "Edge and VolumePairList are incompatible, "
-            "Edge will override whatever pairs VolumePairlist selects."
-        )
     if not conf.get('ask_strategy', {}).get('use_sell_signal', True):
         raise OperationalException(
             "Edge requires `use_sell_signal` to be True, otherwise no sells will happen."

freqtrade/configuration/configuration.py

@@ -11,10 +11,10 @@ from freqtrade import constants
 from freqtrade.configuration.check_exchange import check_exchange
 from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings
 from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir
-from freqtrade.configuration.load_config import load_config_file
+from freqtrade.configuration.load_config import load_config_file, load_file
 from freqtrade.exceptions import OperationalException
 from freqtrade.loggers import setup_logging
-from freqtrade.misc import deep_merge_dicts, json_load
+from freqtrade.misc import deep_merge_dicts
 from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode
 
 
@@ -75,8 +75,6 @@ class Configuration:
         # Normalize config
         if 'internals' not in config:
             config['internals'] = {}
-        # TODO: This can be deleted along with removal of deprecated
-        # experimental settings
         if 'ask_strategy' not in config:
             config['ask_strategy'] = {}
 
@@ -108,6 +106,8 @@ class Configuration:
 
         self._process_plot_options(config)
 
+        self._process_data_options(config)
+
         # Check if the exchange set by the user is supported
         check_exchange(config, config.get('experimental', {}).get('block_bad_exchanges', True))
 
@@ -399,6 +399,11 @@ class Configuration:
         self._args_to_config(config, argname='dataformat_trades',
                              logstring='Using "{}" to store trades data.')
 
+    def _process_data_options(self, config: Dict[str, Any]) -> None:
+
+        self._args_to_config(config, argname='new_pairs_days',
+                             logstring='Detected --new-pairs-days: {}')
+
     def _process_runmode(self, config: Dict[str, Any]) -> None:
 
         self._args_to_config(config, argname='dry_run',
@@ -445,6 +450,7 @@ class Configuration:
         """
 
         if "pairs" in config:
+            config['exchange']['pair_whitelist'] = config['pairs']
             return
 
         if "pairs_file" in self.args and self.args["pairs_file"]:
@@ -454,9 +460,8 @@ class Configuration:
             # or if pairs file is specified explicitely
             if not pairs_file.exists():
                 raise OperationalException(f'No pairs file found with path "{pairs_file}".')
-            with pairs_file.open('r') as f:
-                config['pairs'] = json_load(f)
-                config['pairs'].sort()
+            config['pairs'] = load_file(pairs_file)
+            config['pairs'].sort()
             return
 
         if 'config' in self.args and self.args['config']:
@@ -466,7 +471,6 @@ class Configuration:
             # Fall back to /dl_path/pairs.json
             pairs_file = config['datadir'] / 'pairs.json'
             if pairs_file.exists():
-                with pairs_file.open('r') as f:
-                    config['pairs'] = json_load(f)
+                config['pairs'] = load_file(pairs_file)
                 if 'pairs' in config:
                     config['pairs'].sort()

freqtrade/configuration/directory_operations.py

@@ -24,6 +24,21 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Pat
     return folder
 
 
+def chown_user_directory(directory: Path) -> None:
+    """
+    Use Sudo to change permissions of the home-directory if necessary
+    Only applies when running in docker!
+    """
+    import os
+    if os.environ.get('FT_APP_ENV') == 'docker':
+        try:
+            import subprocess
+            subprocess.check_output(
+                ['sudo', 'chown', '-R', 'ftuser:', str(directory.resolve())])
+        except Exception:
+            logger.warning(f"Could not chown {directory}")
+
+
 def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
     """
     Create userdata directory structure.
@@ -37,6 +52,7 @@ def create_userdata_dir(directory: str, create_dir: bool = False) -> Path:
     sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "logs",
                 "notebooks", "plot", "strategies", ]
     folder = Path(directory)
+    chown_user_directory(folder)
     if not folder.is_dir():
         if create_dir:
             folder.mkdir(parents=True)
@@ -72,6 +88,5 @@ def copy_sample_files(directory: Path, overwrite: bool = False) -> None:
             if not overwrite:
                 logger.warning(f"File `{targetfile}` exists already, not deploying sample file.")
                 continue
-            else:
-                logger.warning(f"File `{targetfile}` exists already, overwriting.")
+            logger.warning(f"File `{targetfile}` exists already, overwriting.")
         shutil.copy(str(sourcedir / source), str(targetfile))

freqtrade/configuration/load_config.py

@@ -38,6 +38,15 @@ def log_config_error_range(path: str, errmsg: str) -> str:
     return ''
 
 
+def load_file(path: Path) -> Dict[str, Any]:
+    try:
+        with path.open('r') as file:
+            config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE)
+    except FileNotFoundError:
+        raise OperationalException(f'File file "{path}" not found!')
+    return config
+
+
 def load_config_file(path: str) -> Dict[str, Any]:
     """
     Loads a config file from the given path

freqtrade/constants.py

@@ -26,7 +26,7 @@ HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss',
 AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList',
                        'AgeFilter', 'PerformanceFilter', 'PrecisionFilter',
                        'PriceFilter', 'RangeStabilityFilter', 'ShuffleFilter',
-                       'SpreadFilter']
+                       'SpreadFilter', 'VolatilityFilter']
 AVAILABLE_PROTECTIONS = ['CooldownPeriod', 'LowProfitPairs', 'MaxDrawdown', 'StoplossGuard']
 AVAILABLE_DATAHANDLERS = ['json', 'jsongz', 'hdf5']
 DRY_RUN_WALLET = 1000
@@ -96,6 +96,7 @@ CONF_SCHEMA = {
     'type': 'object',
     'properties': {
         'max_open_trades': {'type': ['integer', 'number'], 'minimum': -1},
+        'new_pairs_days': {'type': 'integer', 'default': 30},
         'timeframe': {'type': 'string'},
         'stake_currency': {'type': 'string'},
         'stake_amount': {
@@ -176,7 +177,7 @@ CONF_SCHEMA = {
                 'order_book_max': {'type': 'integer', 'minimum': 1, 'maximum': 50},
                 'use_sell_signal': {'type': 'boolean'},
                 'sell_profit_only': {'type': 'boolean'},
-                'sell_profit_offset': {'type': 'number', 'minimum': 0.0},
+                'sell_profit_offset': {'type': 'number'},
                 'ignore_roi_if_buy_signal': {'type': 'boolean'}
             }
         },
@@ -246,14 +247,24 @@ CONF_SCHEMA = {
                 'balance_dust_level': {'type': 'number', 'minimum': 0.0},
                 'notification_settings': {
                     'type': 'object',
+                    'default': {},
                     'properties': {
                         'status': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
                         'warning': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
                         'startup': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
                         'buy': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
-                        'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
                         'buy_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
-                        'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}
+                        'buy_fill': {'type': 'string',
+                                     'enum': TELEGRAM_SETTING_OPTIONS,
+                                     'default': 'off'
+                                     },
+                        'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS},
+                        'sell_fill': {
+                            'type': 'string',
+                            'enum': TELEGRAM_SETTING_OPTIONS,
+                            'default': 'off'
+                            },
                     }
                 }
             },

freqtrade/data/btanalysis.py

@@ -337,7 +337,7 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     """
     Adds a column `col_name` with the cumulative profit for the given trades array.
     :param df: DataFrame with date index
-    :param trades: DataFrame containing trades (requires columns close_date and profit_ratio)
+    :param trades: DataFrame containing trades (requires columns close_date and profit_abs)
     :param col_name: Column name that will be assigned the results
     :param timeframe: Timeframe used during the operations
     :return: Returns df with one additional column, col_name, containing the cumulative profit.
@@ -349,8 +349,8 @@ def create_cum_profit(df: pd.DataFrame, trades: pd.DataFrame, col_name: str,
     timeframe_minutes = timeframe_to_minutes(timeframe)
     # Resample to timeframe to make sure trades match candles
     _trades_sum = trades.resample(f'{timeframe_minutes}min', on='close_date'
-                                  )[['profit_ratio']].sum()
-    df.loc[:, col_name] = _trades_sum['profit_ratio'].cumsum()
+                                  )[['profit_abs']].sum()
+    df.loc[:, col_name] = _trades_sum['profit_abs'].cumsum()
     # Set first value to 0
     df.loc[df.iloc[0].name, col_name] = 0
     # FFill to get continuous

freqtrade/data/converter.py

@@ -110,22 +110,35 @@ def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str)
     df.reset_index(inplace=True)
     len_before = len(dataframe)
     len_after = len(df)
+    pct_missing = (len_after - len_before) / len_before if len_before > 0 else 0
     if len_before != len_after:
-        logger.info(f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}")
+        message = (f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}"
+                   f" - {round(pct_missing * 100, 2)}%")
+        if pct_missing > 0.01:
+            logger.info(message)
+        else:
+            # Don't be verbose if only a small amount is missing
+            logger.debug(message)
     return df
 
 
-def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date') -> DataFrame:
+def trim_dataframe(df: DataFrame, timerange, df_date_col: str = 'date',
+                   startup_candles: int = 0) -> DataFrame:
     """
     Trim dataframe based on given timerange
     :param df: Dataframe to trim
     :param timerange: timerange (use start and end date if available)
-    :param: df_date_col: Column in the dataframe to use as Date column
+    :param df_date_col: Column in the dataframe to use as Date column
+    :param startup_candles: When not 0, is used instead the timerange start date
     :return: trimmed dataframe
     """
-    if timerange.starttype == 'date':
-        start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
-        df = df.loc[df[df_date_col] >= start, :]
+    if startup_candles:
+        # Trim candles instead of timeframe in case of given startup_candle count
+        df = df.iloc[startup_candles:, :]
+    else:
+        if timerange.starttype == 'date':
+            start = datetime.fromtimestamp(timerange.startts, tz=timezone.utc)
+            df = df.loc[df[df_date_col] >= start, :]
     if timerange.stoptype == 'date':
         stop = datetime.fromtimestamp(timerange.stopts, tz=timezone.utc)
         df = df.loc[df[df_date_col] <= stop, :]

freqtrade/data/dataprovider.py

@@ -170,6 +170,6 @@ class DataProvider:
         """
 
         if self._pairlists:
-            return self._pairlists.whitelist
+            return self._pairlists.whitelist.copy()
         else:
             raise OperationalException("Dataprovider was not initialized with a pairlist provider.")

freqtrade/data/history/hdf5datahandler.py

@@ -89,7 +89,7 @@ class HDF5DataHandler(IDataHandler):
             if timerange.starttype == 'date':
                 where.append(f"date >= Timestamp({timerange.startts * 1e9})")
             if timerange.stoptype == 'date':
-                where.append(f"date < Timestamp({timerange.stopts * 1e9})")
+                where.append(f"date <= Timestamp({timerange.stopts * 1e9})")
 
         pairdata = pd.read_hdf(filename, key=key, mode="r", where=where)
 

freqtrade/data/history/history_utils.py

@@ -155,6 +155,7 @@ def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optiona
 def _download_pair_history(datadir: Path,
                            exchange: Exchange,
                            pair: str, *,
+                           new_pairs_days: int = 30,
                            timeframe: str = '5m',
                            timerange: Optional[TimeRange] = None,
                            data_handler: IDataHandler = None) -> bool:
@@ -193,7 +194,7 @@ def _download_pair_history(datadir: Path,
                                                timeframe=timeframe,
                                                since_ms=since_ms if since_ms else
                                                int(arrow.utcnow().shift(
-                                                   days=-30).float_timestamp) * 1000
+                                                   days=-new_pairs_days).float_timestamp) * 1000
                                                )
         # TODO: Maybe move parsing to exchange class (?)
         new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair,
@@ -223,7 +224,8 @@ def _download_pair_history(datadir: Path,
 
 def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str],
                                 datadir: Path, timerange: Optional[TimeRange] = None,
-                                erase: bool = False, data_format: str = None) -> List[str]:
+                                new_pairs_days: int = 30, erase: bool = False,
+                                data_format: str = None) -> List[str]:
     """
     Refresh stored ohlcv data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -246,12 +248,14 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes
             logger.info(f'Downloading pair {pair}, interval {timeframe}.')
             _download_pair_history(datadir=datadir, exchange=exchange,
                                    pair=pair, timeframe=str(timeframe),
+                                   new_pairs_days=new_pairs_days,
                                    timerange=timerange, data_handler=data_handler)
     return pairs_not_available
 
 
 def _download_trades_history(exchange: Exchange,
                              pair: str, *,
+                             new_pairs_days: int = 30,
                              timerange: Optional[TimeRange] = None,
                              data_handler: IDataHandler
                              ) -> bool:
@@ -263,7 +267,7 @@ def _download_trades_history(exchange: Exchange,
 
         since = timerange.startts * 1000 if \
             (timerange and timerange.starttype == 'date') else int(arrow.utcnow().shift(
-                days=-30).float_timestamp) * 1000
+                days=-new_pairs_days).float_timestamp) * 1000
 
         trades = data_handler.trades_load(pair)
 
@@ -311,8 +315,8 @@ def _download_trades_history(exchange: Exchange,
 
 
 def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
-                                 timerange: TimeRange, erase: bool = False,
-                                 data_format: str = 'jsongz') -> List[str]:
+                                 timerange: TimeRange, new_pairs_days: int = 30,
+                                 erase: bool = False, data_format: str = 'jsongz') -> List[str]:
     """
     Refresh stored trades data for backtesting and hyperopt operations.
     Used by freqtrade download-data subcommand.
@@ -333,6 +337,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
         logger.info(f'Downloading trades for pair {pair}.')
         _download_trades_history(exchange=exchange,
                                  pair=pair,
+                                 new_pairs_days=new_pairs_days,
                                  timerange=timerange,
                                  data_handler=data_handler)
     return pairs_not_available

freqtrade/edge/edge_positioning.py

@@ -81,12 +81,16 @@ class Edge:
         if config.get('fee'):
             self.fee = config['fee']
         else:
-            self.fee = self.exchange.get_fee(symbol=expand_pairlist(
-                self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
+            try:
+                self.fee = self.exchange.get_fee(symbol=expand_pairlist(
+                    self.config['exchange']['pair_whitelist'], list(self.exchange.markets))[0])
+            except IndexError:
+                self.fee = None
+
+    def calculate(self, pairs: List[str]) -> bool:
+        if self.fee is None and pairs:
+            self.fee = self.exchange.get_fee(pairs[0])
 
-    def calculate(self) -> bool:
-        pairs = expand_pairlist(self.config['exchange']['pair_whitelist'],
-                                list(self.exchange.markets))
         heartbeat = self.edge_config.get('process_throttle_secs')
 
         if (self._last_updated > 0) and (

freqtrade/exchange/__init__.py

@@ -8,10 +8,11 @@ from freqtrade.exchange.binance import Binance
 from freqtrade.exchange.bittrex import Bittrex
 from freqtrade.exchange.bybit import Bybit
 from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges,
-                                         get_exchange_bad_reason, is_exchange_bad,
                                          is_exchange_known_ccxt, is_exchange_officially_supported,
                                          market_is_active, timeframe_to_minutes, timeframe_to_msecs,
                                          timeframe_to_next_date, timeframe_to_prev_date,
-                                         timeframe_to_seconds)
+                                         timeframe_to_seconds, validate_exchange,
+                                         validate_exchanges)
 from freqtrade.exchange.ftx import Ftx
 from freqtrade.exchange.kraken import Kraken
+from freqtrade.exchange.kucoin import Kucoin

freqtrade/exchange/binance.py

@@ -52,7 +52,7 @@ class Binance(Exchange):
                 'In stoploss limit order, stop price should be more than limit price')
 
         if self._config['dry_run']:
-            dry_order = self.dry_run_order(
+            dry_order = self.create_dry_run_order(
                 pair, ordertype, "sell", amount, stop_price)
             return dry_order
 

freqtrade/exchange/bittrex.py

@@ -12,10 +12,6 @@ class Bittrex(Exchange):
     """
     Bittrex exchange class. Contains adjustments needed for Freqtrade to work
     with this exchange.
-
-    Please note that this exchange is not included in the list of exchanges
-    officially supported by the Freqtrade development team. So some features
-    may still not work as expected.
     """
 
     _ft_has: Dict = {

freqtrade/exchange/common.py

@@ -18,78 +18,8 @@ BAD_EXCHANGES = {
     "bitmex": "Various reasons.",
     "bitstamp": "Does not provide history. "
                 "Details in https://github.com/freqtrade/freqtrade/issues/1983",
-    "hitbtc": "This API cannot be used with Freqtrade. "
-              "Use `hitbtc2` exchange id to access this exchange.",
     "phemex": "Does not provide history. ",
     "poloniex": "Does not provide fetch_order endpoint to fetch both open and closed orders.",
-    **dict.fromkeys([
-        'adara',
-        'anxpro',
-        'bigone',
-        'coinbase',
-        'coinexchange',
-        'coinmarketcap',
-        'lykke',
-        'xbtce',
-    ], "Does not provide timeframes. ccxt fetchOHLCV: False"),
-    **dict.fromkeys([
-        'bcex',
-        'bit2c',
-        'bitbay',
-        'bitflyer',
-        'bitforex',
-        'bithumb',
-        'bitso',
-        'bitstamp1',
-        'bl3p',
-        'braziliex',
-        'btcbox',
-        'btcchina',
-        'btctradeim',
-        'btctradeua',
-        'bxinth',
-        'chilebit',
-        'coincheck',
-        'coinegg',
-        'coinfalcon',
-        'coinfloor',
-        'coingi',
-        'coinmate',
-        'coinone',
-        'coinspot',
-        'coolcoin',
-        'crypton',
-        'deribit',
-        'exmo',
-        'exx',
-        'flowbtc',
-        'foxbit',
-        'fybse',
-        # 'hitbtc',
-        'ice3x',
-        'independentreserve',
-        'indodax',
-        'itbit',
-        'lakebtc',
-        'latoken',
-        'liquid',
-        'livecoin',
-        'luno',
-        'mixcoins',
-        'negociecoins',
-        'nova',
-        'paymium',
-        'southxchange',
-        'stronghold',
-        'surbitcoin',
-        'therock',
-        'tidex',
-        'vaultoro',
-        'vbtc',
-        'virwox',
-        'yobit',
-        'zaif',
-    ], "Does not provide timeframes. ccxt fetchOHLCV: emulated"),
 }
 
 MAP_EXCHANGE_CHILDCLASS = {
@@ -98,6 +28,29 @@ MAP_EXCHANGE_CHILDCLASS = {
 }
 
 
+EXCHANGE_HAS_REQUIRED = [
+    # Required / private
+    'fetchOrder',
+    'cancelOrder',
+    'createOrder',
+    # 'createLimitOrder', 'createMarketOrder',
+    'fetchBalance',
+
+    # Public endpoints
+    'loadMarkets',
+    'fetchOHLCV',
+]
+
+EXCHANGE_HAS_OPTIONAL = [
+    # Private
+    'fetchMyTrades',  # Trades for order - fee detection
+    # Public
+    'fetchOrderBook', 'fetchL2OrderBook', 'fetchTicker',  # OR for pricing
+    'fetchTickers',  # For volumepairlist?
+    'fetchTrades',  # Downloading trades data
+]
+
+
 def calculate_backoff(retrycount, max_retries):
     """
     Calculate backoff
@@ -140,7 +93,7 @@ def retrier(_func=None, retries=API_RETRY_COUNT):
                     logger.warning('retrying %s() still for %s times', f.__name__, count)
                     count -= 1
                     kwargs.update({'count': count})
-                    if isinstance(ex, DDosProtection) or isinstance(ex, RetryableOrderError):
+                    if isinstance(ex, (DDosProtection, RetryableOrderError)):
                         # increasing backoff
                         backoff_delay = calculate_backoff(count + 1, retries)
                         logger.info(f"Applying DDosProtection backoff delay: {backoff_delay}")

freqtrade/exchange/exchange.py

@@ -14,6 +14,7 @@ from typing import Any, Dict, List, Optional, Tuple
 import arrow
 import ccxt
 import ccxt.async_support as ccxt_async
+from cachetools import TTLCache
 from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE,
                                             decimal_to_precision)
 from pandas import DataFrame
@@ -23,7 +24,8 @@ from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list
 from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError,
                                   InvalidOrderException, OperationalException, RetryableOrderError,
                                   TemporaryError)
-from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, retrier,
+from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES,
+                                       EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier,
                                        retrier_async)
 from freqtrade.misc import deep_merge_dicts, safe_value_fallback2
 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
@@ -62,6 +64,7 @@ class Exchange:
         "trades_pagination": "time",  # Possible are "time" or "id"
         "trades_pagination_arg": "since",
         "l2_limit_range": None,
+        "l2_limit_range_required": True,  # Allow Empty L2 limit (kucoin)
     }
     _ft_has: Dict = {}
 
@@ -82,6 +85,9 @@ class Exchange:
         # Timestamp of last markets refresh
         self._last_markets_refresh: int = 0
 
+        # Cache for 10 minutes ...
+        self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10)
+
         # Holds candles
         self._klines: Dict[Tuple[str, str], DataFrame] = {}
 
@@ -357,7 +363,6 @@ class Exchange:
         invalid_pairs = []
         for pair in extended_pairs:
             # Note: ccxt has BaseCurrency/QuoteCurrency format for pairs
-            # TODO: add a support for having coins in BTC/USDT format
             if self.markets and pair not in self.markets:
                 raise OperationalException(
                     f'Pair {pair} is not available on {self.name}. '
@@ -533,7 +538,9 @@ class Exchange:
         # reserve some percent defined in config (5% default) + stoploss
         amount_reserve_percent = 1.0 + self._config.get('amount_reserve_percent',
                                                         DEFAULT_AMOUNT_RESERVE_PERCENT)
-        amount_reserve_percent += abs(stoploss)
+        amount_reserve_percent = (
+          amount_reserve_percent / (1 - abs(stoploss)) if abs(stoploss) != 1 else 1.5
+        )
         # it should not be more than 50%
         amount_reserve_percent = max(min(amount_reserve_percent, 1.5), 1)
 
@@ -542,8 +549,8 @@ class Exchange:
         # See also #2575 at github.
         return max(min_stake_amounts) * amount_reserve_percent
 
-    def dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
-                      rate: float, params: Dict = {}) -> Dict[str, Any]:
+    def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float,
+                             rate: float, params: Dict = {}) -> Dict[str, Any]:
         order_id = f'dry_run_{side}_{datetime.now().timestamp()}'
         _amount = self.amount_to_precision(pair, amount)
         dry_order = {
@@ -617,7 +624,7 @@ class Exchange:
             rate: float, time_in_force: str) -> Dict:
 
         if self._config['dry_run']:
-            dry_order = self.dry_run_order(pair, ordertype, "buy", amount, rate)
+            dry_order = self.create_dry_run_order(pair, ordertype, "buy", amount, rate)
             return dry_order
 
         params = self._params.copy()
@@ -630,7 +637,7 @@ class Exchange:
              rate: float, time_in_force: str = 'gtc') -> Dict:
 
         if self._config['dry_run']:
-            dry_order = self.dry_run_order(pair, ordertype, "sell", amount, rate)
+            dry_order = self.create_dry_run_order(pair, ordertype, "sell", amount, rate)
             return dry_order
 
         params = self._params.copy()
@@ -659,23 +666,8 @@ class Exchange:
 
         raise OperationalException(f"stoploss is not implemented for {self.name}.")
 
-    @retrier
-    def get_balance(self, currency: str) -> float:
-        if self._config['dry_run']:
-            return self._config['dry_run_wallet']
-
-        # ccxt exception is already handled by get_balances
-        balances = self.get_balances()
-        balance = balances.get(currency)
-        if balance is None:
-            raise TemporaryError(
-                f'Could not get {currency} balance due to malformed exchange response: {balances}')
-        return balance['free']
-
     @retrier
     def get_balances(self) -> dict:
-        if self._config['dry_run']:
-            return {}
 
         try:
             balances = self._api.fetch_balance()
@@ -695,9 +687,19 @@ class Exchange:
             raise OperationalException(e) from e
 
     @retrier
-    def get_tickers(self) -> Dict:
+    def get_tickers(self, cached: bool = False) -> Dict:
+        """
+        :param cached: Allow cached result
+        :return: fetch_tickers result
+        """
+        if cached:
+            tickers = self._fetch_tickers_cache.get('fetch_tickers')
+            if tickers:
+                return tickers
         try:
-            return self._api.fetch_tickers()
+            tickers = self._api.fetch_tickers()
+            self._fetch_tickers_cache['fetch_tickers'] = tickers
+            return tickers
         except ccxt.NotSupported as e:
             raise OperationalException(
                 f'Exchange {self._api.name} does not support fetching tickers in batch. '
@@ -806,7 +808,7 @@ class Exchange:
 
         # Gather coroutines to run
         for pair, timeframe in set(pair_list):
-            if (not ((pair, timeframe) in self._klines)
+            if (((pair, timeframe) not in self._klines)
                     or self._now_is_time_to_refresh(pair, timeframe)):
                 input_coroutines.append(self._async_get_candle_history(pair, timeframe,
                                                                        since_ms=since_ms))
@@ -958,7 +960,7 @@ class Exchange:
         while True:
             t = await self._async_fetch_trades(pair,
                                                params={self._trades_pagination_arg: from_id})
-            if len(t):
+            if t:
                 # Skip last id since its the key for the next call
                 trades.extend(t[:-1])
                 if from_id == t[-1][1] or t[-1][0] > until:
@@ -990,7 +992,7 @@ class Exchange:
         # DEFAULT_TRADES_COLUMNS: 1 -> id
         while True:
             t = await self._async_fetch_trades(pair, since=since)
-            if len(t):
+            if t:
                 since = t[-1][0]
                 trades.extend(t)
                 # Reached the end of the defined-download period
@@ -1157,14 +1159,20 @@ class Exchange:
         return self.fetch_order(order_id, pair)
 
     @staticmethod
-    def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]]):
+    def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]],
+                               range_required: bool = True):
         """
         Get next greater value in the list.
         Used by fetch_l2_order_book if the api only supports a limited range
         """
         if not limit_range:
             return limit
-        return min([x for x in limit_range if limit <= x] + [max(limit_range)])
+
+        result = min([x for x in limit_range if limit <= x] + [max(limit_range)])
+        if not range_required and limit > result:
+            # Range is not required - we can use None as parameter.
+            return None
+        return result
 
     @retrier
     def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict:
@@ -1174,7 +1182,8 @@ class Exchange:
         Returns a dict in the format
         {'asks': [price, volume], 'bids': [price, volume]}
         """
-        limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'])
+        limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'],
+                                             self._ft_has['l2_limit_range_required'])
         try:
 
             return self._api.fetch_l2_order_book(pair, limit1)
@@ -1306,14 +1315,6 @@ class Exchange:
                 self.calculate_fee_rate(order))
 
 
-def is_exchange_bad(exchange_name: str) -> bool:
-    return exchange_name in BAD_EXCHANGES
-
-
-def get_exchange_bad_reason(exchange_name: str) -> str:
-    return BAD_EXCHANGES.get(exchange_name, "")
-
-
 def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool:
     return exchange_name in ccxt_exchanges(ccxt_module)
 
@@ -1334,7 +1335,36 @@ def available_exchanges(ccxt_module: CcxtModuleType = None) -> List[str]:
     Return exchanges available to the bot, i.e. non-bad exchanges in the ccxt list
     """
     exchanges = ccxt_exchanges(ccxt_module)
-    return [x for x in exchanges if not is_exchange_bad(x)]
+    return [x for x in exchanges if validate_exchange(x)[0]]
+
+
+def validate_exchange(exchange: str) -> Tuple[bool, str]:
+    ex_mod = getattr(ccxt, exchange.lower())()
+    if not ex_mod or not ex_mod.has:
+        return False, ''
+    missing = [k for k in EXCHANGE_HAS_REQUIRED if ex_mod.has.get(k) is not True]
+    if missing:
+        return False, f"missing: {', '.join(missing)}"
+
+    missing_opt = [k for k in EXCHANGE_HAS_OPTIONAL if not ex_mod.has.get(k)]
+
+    if exchange.lower() in BAD_EXCHANGES:
+        return False, BAD_EXCHANGES.get(exchange.lower(), '')
+    if missing_opt:
+        return True, f"missing opt: {', '.join(missing_opt)}"
+
+    return True, ''
+
+
+def validate_exchanges(all_exchanges: bool) -> List[Tuple[str, bool, str]]:
+    """
+    :return: List of tuples with exchangename, valid, reason.
+    """
+    exchanges = ccxt_exchanges() if all_exchanges else available_exchanges()
+    exchanges_valid = [
+        (e, *validate_exchange(e)) for e in exchanges
+    ]
+    return exchanges_valid
 
 
 def timeframe_to_seconds(timeframe: str) -> int:

freqtrade/exchange/ftx.py

@@ -53,7 +53,7 @@ class Ftx(Exchange):
         stop_price = self.price_to_precision(pair, stop_price)
 
         if self._config['dry_run']:
-            dry_order = self.dry_run_order(
+            dry_order = self.create_dry_run_order(
                 pair, ordertype, "sell", amount, stop_price)
             return dry_order
 
@@ -63,10 +63,11 @@ class Ftx(Exchange):
                 # set orderPrice to place limit order, otherwise it's a market order
                 params['orderPrice'] = limit_rate
 
+            params['stopPrice'] = stop_price
             amount = self.amount_to_precision(pair, amount)
 
             order = self._api.create_order(symbol=pair, type=ordertype, side='sell',
-                                           amount=amount, price=stop_price, params=params)
+                                           amount=amount, params=params)
             logger.info('stoploss order added for %s. '
                         'stop price: %s.', pair, stop_price)
             return order

freqtrade/exchange/kraken.py

@@ -92,7 +92,7 @@ class Kraken(Exchange):
         stop_price = self.price_to_precision(pair, stop_price)
 
         if self._config['dry_run']:
-            dry_order = self.dry_run_order(
+            dry_order = self.create_dry_run_order(
                 pair, ordertype, "sell", amount, stop_price)
             return dry_order
 

freqtrade/exchange/kucoin.py

@@ -0,0 +1,24 @@
+""" Kucoin exchange subclass """
+import logging
+from typing import Dict
+
+from freqtrade.exchange import Exchange
+
+
+logger = logging.getLogger(__name__)
+
+
+class Kucoin(Exchange):
+    """
+    Kucoin exchange class. Contains adjustments needed for Freqtrade to work
+    with this exchange.
+
+    Please note that this exchange is not included in the list of exchanges
+    officially supported by the Freqtrade development team. So some features
+    may still not work as expected.
+    """
+
+    _ft_has: Dict = {
+        "l2_limit_range": [20, 100],
+        "l2_limit_range_required": False,
+    }

freqtrade/freqtradebot.py

@@ -113,7 +113,7 @@ class FreqtradeBot(LoggingMixin):
         via RPC about changes in the bot status.
         """
         self.rpc.send_msg({
-            'type': RPCMessageType.STATUS_NOTIFICATION,
+            'type': RPCMessageType.STATUS,
             'status': msg
         })
 
@@ -187,7 +187,7 @@ class FreqtradeBot(LoggingMixin):
         if self.get_free_open_trades():
             self.enter_positions()
 
-        Trade.session.flush()
+        Trade.query.session.flush()
 
     def process_stopped(self) -> None:
         """
@@ -205,7 +205,7 @@ class FreqtradeBot(LoggingMixin):
 
         if len(open_trades) != 0:
             msg = {
-                'type': RPCMessageType.WARNING_NOTIFICATION,
+                'type': RPCMessageType.WARNING,
                 'status':  f"{len(open_trades)} open trades active.\n\n"
                            f"Handle these trades manually on {self.exchange.name}, "
                            f"or '/start' the bot again and use '/stopbuy' "
@@ -225,7 +225,7 @@ class FreqtradeBot(LoggingMixin):
 
         # Calculating Edge positioning
         if self.edge:
-            self.edge.calculate()
+            self.edge.calculate(_whitelist)
             _whitelist = self.edge.adjust(_whitelist)
 
         if trades:
@@ -378,7 +378,7 @@ class FreqtradeBot(LoggingMixin):
             if lock:
                 self.log_once(f"Global pairlock active until "
                               f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}. "
-                              "Not creating new trades.", logger.info)
+                              f"Not creating new trades, reason: {lock.reason}.", logger.info)
             else:
                 self.log_once("Global pairlock active. Not creating new trades.", logger.info)
             return trades_created
@@ -410,9 +410,7 @@ class FreqtradeBot(LoggingMixin):
 
         bid_strategy = self.config.get('bid_strategy', {})
         if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False):
-            logger.info(
-                f"Getting price from order book {bid_strategy['price_side'].capitalize()} side."
-            )
+
             order_book_top = bid_strategy.get('order_book_top', 1)
             order_book = self.exchange.fetch_l2_order_book(pair, order_book_top)
             logger.debug('order_book %s', order_book)
@@ -425,7 +423,8 @@ class FreqtradeBot(LoggingMixin):
                     f"Orderbook: {order_book}"
                  )
                 raise PricingError from e
-            logger.info(f'...top {order_book_top} order book buy rate {rate_from_l2:.8f}')
+            logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side "
+                        f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}")
             used_rate = rate_from_l2
         else:
             logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price")
@@ -457,7 +456,8 @@ class FreqtradeBot(LoggingMixin):
             lock = PairLocks.get_pair_longest_lock(pair, nowtime)
             if lock:
                 self.log_once(f"Pair {pair} is still locked until "
-                              f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)}.",
+                              f"{lock.lock_end_time.strftime(constants.DATETIME_PRINT_FORMAT)} "
+                              f"due to {lock.reason}.",
                               logger.info)
             else:
                 self.log_once(f"Pair {pair} is still locked.", logger.info)
@@ -473,25 +473,22 @@ class FreqtradeBot(LoggingMixin):
         (buy, sell) = self.strategy.get_signal(pair, self.strategy.timeframe, analyzed_df)
 
         if buy and not sell:
-            stake_amount = self.wallets.get_trade_stake_amount(pair, self.get_free_open_trades(),
-                                                               self.edge)
+            stake_amount = self.wallets.get_trade_stake_amount(pair, self.edge)
             if not stake_amount:
                 logger.debug(f"Stake amount is 0, ignoring possible trade for {pair}.")
                 return False
 
-            logger.info(f"Buy signal found: about create a new trade with stake_amount: "
+            logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: "
                         f"{stake_amount} ...")
 
             bid_check_dom = self.config.get('bid_strategy', {}).get('check_depth_of_market', {})
             if ((bid_check_dom.get('enabled', False)) and
                     (bid_check_dom.get('bids_to_ask_delta', 0) > 0)):
                 if self._check_depth_of_market_buy(pair, bid_check_dom):
-                    logger.info(f'Executing Buy for {pair}.')
                     return self.execute_buy(pair, stake_amount)
                 else:
                     return False
 
-            logger.info(f'Executing Buy for {pair}')
             return self.execute_buy(pair, stake_amount)
         else:
             return False
@@ -621,8 +618,8 @@ class FreqtradeBot(LoggingMixin):
         if order_status == 'closed':
             self.update_trade_state(trade, order_id, order)
 
-        Trade.session.add(trade)
-        Trade.session.flush()
+        Trade.query.session.add(trade)
+        Trade.query.session.flush()
 
         # Updating wallets
         self.wallets.update()
@@ -637,7 +634,7 @@ class FreqtradeBot(LoggingMixin):
         """
         msg = {
             'trade_id': trade.id,
-            'type': RPCMessageType.BUY_NOTIFICATION,
+            'type': RPCMessageType.BUY,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'limit': trade.open_rate,
@@ -661,7 +658,7 @@ class FreqtradeBot(LoggingMixin):
 
         msg = {
             'trade_id': trade.id,
-            'type': RPCMessageType.BUY_CANCEL_NOTIFICATION,
+            'type': RPCMessageType.BUY_CANCEL,
             'exchange': self.exchange.name.capitalize(),
             'pair': trade.pair,
             'limit': trade.open_rate,
@@ -678,6 +675,21 @@ class FreqtradeBot(LoggingMixin):
         # Send the message
         self.rpc.send_msg(msg)
 
+    def _notify_buy_fill(self, trade: Trade) -> None:
+        msg = {
+            'trade_id': trade.id,
+            'type': RPCMessageType.BUY_FILL,
+            'exchange': self.exchange.name.capitalize(),
+            'pair': trade.pair,
+            'open_rate': trade.open_rate,
+            'stake_amount': trade.stake_amount,
+            'stake_currency': self.config['stake_currency'],
+            'fiat_currency': self.config.get('fiat_display_currency', None),
+            'amount': trade.amount,
+            'open_date': trade.open_date,
+        }
+        self.rpc.send_msg(msg)
+
 #
 # SELL / exit positions / close trades logic and methods
 #
@@ -1205,7 +1217,7 @@ class FreqtradeBot(LoggingMixin):
         # In case of market sell orders the order can be closed immediately
         if order.get('status', 'unknown') == 'closed':
             self.update_trade_state(trade, trade.open_order_id, order)
-        Trade.session.flush()
+        Trade.query.session.flush()
 
         # Lock pair for one candle to prevent immediate rebuys
         self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc),
@@ -1215,19 +1227,20 @@ class FreqtradeBot(LoggingMixin):
 
         return True
 
-    def _notify_sell(self, trade: Trade, order_type: str) -> None:
+    def _notify_sell(self, trade: Trade, order_type: str, fill: bool = False) -> None:
         """
         Sends rpc notification when a sell occured.
         """
         profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested
         profit_trade = trade.calc_profit(rate=profit_rate)
         # Use cached rates here - it was updated seconds ago.
-        current_rate = self.get_sell_rate(trade.pair, False)
+        current_rate = self.get_sell_rate(trade.pair, False) if not fill else None
         profit_ratio = trade.calc_profit_ratio(profit_rate)
         gain = "profit" if profit_ratio > 0 else "loss"
 
         msg = {
-            'type': RPCMessageType.SELL_NOTIFICATION,
+            'type': (RPCMessageType.SELL_FILL if fill
+                     else RPCMessageType.SELL),
             'trade_id': trade.id,
             'exchange': trade.exchange.capitalize(),
             'pair': trade.pair,
@@ -1236,6 +1249,7 @@ class FreqtradeBot(LoggingMixin):
             'order_type': order_type,
             'amount': trade.amount,
             'open_rate': trade.open_rate,
+            'close_rate': trade.close_rate,
             'current_rate': current_rate,
             'profit_amount': profit_trade,
             'profit_ratio': profit_ratio,
@@ -1270,7 +1284,7 @@ class FreqtradeBot(LoggingMixin):
         gain = "profit" if profit_ratio > 0 else "loss"
 
         msg = {
-            'type': RPCMessageType.SELL_CANCEL_NOTIFICATION,
+            'type': RPCMessageType.SELL_CANCEL,
             'trade_id': trade.id,
             'exchange': trade.exchange.capitalize(),
             'pair': trade.pair,
@@ -1347,9 +1361,15 @@ class FreqtradeBot(LoggingMixin):
 
         # Updating wallets when order is closed
         if not trade.is_open:
+            if not stoploss_order and not trade.open_order_id:
+                self._notify_sell(trade, '', True)
             self.protections.stop_per_pair(trade.pair)
             self.protections.global_stop()
             self.wallets.update()
+        elif not trade.open_order_id:
+            # Buy fill
+            self._notify_buy_fill(trade)
+
         return False
 
     def apply_fee_conditional(self, trade: Trade, trade_base_currency: str,

freqtrade/misc.py

@@ -81,7 +81,7 @@ def json_load(datafile: IO) -> Any:
     """
     load data with rapidjson
     Use this to have a consistent experience,
-    sete number_mode to "NM_NATIVE" for greatest speed
+    set number_mode to "NM_NATIVE" for greatest speed
     """
     return rapidjson.load(datafile, number_mode=rapidjson.NM_NATIVE)
 

freqtrade/optimize/backtesting.py

@@ -239,7 +239,7 @@ class Backtesting:
                 # Use the maximum between close_rate and low as we
                 # cannot sell outside of a candle.
                 # Applies when a new ROI setting comes in place and the whole candle is above that.
-                return max(close_rate, sell_row[LOW_IDX])
+                return min(max(close_rate, sell_row[LOW_IDX]), sell_row[HIGH_IDX])
 
             else:
                 # This should not be reached...
@@ -273,11 +273,9 @@ class Backtesting:
 
         return None
 
-    def _enter_trade(self, pair: str, row: List, max_open_trades: int,
-                     open_trade_count: int) -> Optional[LocalTrade]:
+    def _enter_trade(self, pair: str, row: List) -> Optional[LocalTrade]:
         try:
-            stake_amount = self.wallets.get_trade_stake_amount(
-                pair, max_open_trades - open_trade_count, None)
+            stake_amount = self.wallets.get_trade_stake_amount(pair, None)
         except DependencyException:
             return None
         min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, row[OPEN_IDX], -0.05)
@@ -354,7 +352,7 @@ class Backtesting:
         data: Dict = self._get_ohlcv_as_lists(processed)
 
         # Indexes per pair, so some pairs are allowed to have a missing start.
-        indexes: Dict = {}
+        indexes: Dict = defaultdict(int)
         tmp = start_date + timedelta(minutes=self.timeframe_min)
 
         open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
@@ -365,9 +363,6 @@ class Backtesting:
             open_trade_count_start = open_trade_count
 
             for i, pair in enumerate(data):
-                if pair not in indexes:
-                    indexes[pair] = 0
-
                 try:
                     row = data[pair][indexes[pair]]
                 except IndexError:
@@ -388,7 +383,7 @@ class Backtesting:
                         and tmp != end_date
                         and row[BUY_IDX] == 1 and row[SELL_IDX] != 1
                         and not PairLocks.is_pair_locked(pair, row[DATE_IDX])):
-                    trade = self._enter_trade(pair, row, max_open_trades, open_trade_count_start)
+                    trade = self._enter_trade(pair, row)
                     if trade:
                         # TODO: hacky workaround to avoid opening > max_open_trades
                         # This emulates previous behaviour - not sure if this is correct
@@ -443,7 +438,8 @@ class Backtesting:
 
         # Trim startup period from analyzed dataframe
         for pair, df in preprocessed.items():
-            preprocessed[pair] = trim_dataframe(df, timerange)
+            preprocessed[pair] = trim_dataframe(df, timerange,
+                                                startup_candles=self.required_startup)
         min_date, max_date = history.get_timerange(preprocessed)
 
         logger.info(f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
@@ -477,6 +473,7 @@ class Backtesting:
         data: Dict[str, Any] = {}
 
         data, timerange = self.load_bt_data()
+        logger.info("Dataload complete. Calculating indicators")
 
         for strat in self.strategylist:
             min_date, max_date = self.backtest_one_strategy(strat, data, timerange)

freqtrade/optimize/edge_cli.py

@@ -44,7 +44,7 @@ class EdgeCli:
             'timerange') is None else str(self.config.get('timerange')))
 
     def start(self) -> None:
-        result = self.edge.calculate()
+        result = self.edge.calculate(self.config['exchange']['pair_whitelist'])
         if result:
             print('')  # blank line for readability
             print(generate_edge_table(self.edge._cached_pairs))

freqtrade/optimize/hyperopt.py

@@ -26,6 +26,7 @@ from freqtrade.data.history import get_timerange
 from freqtrade.misc import file_dump_json, plural
 from freqtrade.optimize.backtesting import Backtesting
 # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules
+from freqtrade.optimize.hyperopt_auto import HyperOptAuto
 from freqtrade.optimize.hyperopt_interface import IHyperOpt  # noqa: F401
 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss  # noqa: F401
 from freqtrade.optimize.hyperopt_tools import HyperoptTools
@@ -61,14 +62,18 @@ class Hyperopt:
     hyperopt = Hyperopt(config)
     hyperopt.start()
     """
+    custom_hyperopt: IHyperOpt
 
     def __init__(self, config: Dict[str, Any]) -> None:
         self.config = config
 
         self.backtesting = Backtesting(self.config)
 
-        self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
-        self.custom_hyperopt.__class__.strategy = self.backtesting.strategy
+        if not self.config.get('hyperopt'):
+            self.custom_hyperopt = HyperOptAuto(self.config)
+        else:
+            self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config)
+        self.custom_hyperopt.strategy = self.backtesting.strategy
 
         self.custom_hyperoptloss = HyperOptLossResolver.load_hyperoptloss(self.config)
         self.calculate_loss = self.custom_hyperoptloss.hyperopt_loss_function
@@ -374,12 +379,13 @@ class Hyperopt:
         logger.info(f"Using optimizer random state: {self.random_state}")
         self.hyperopt_table_header = -1
         data, timerange = self.backtesting.load_bt_data()
-
+        logger.info("Dataload complete. Calculating indicators")
         preprocessed = self.backtesting.strategy.ohlcvdata_to_dataframe(data)
 
         # Trim startup period from analyzed dataframe
         for pair, df in preprocessed.items():
-            preprocessed[pair] = trim_dataframe(df, timerange)
+            preprocessed[pair] = trim_dataframe(df, timerange,
+                                                startup_candles=self.backtesting.required_startup)
         min_date, max_date = get_timerange(preprocessed)
 
         logger.info(f'Hyperopting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '

freqtrade/optimize/hyperopt_auto.py

@@ -0,0 +1,89 @@
+"""
+HyperOptAuto class.
+This module implements a convenience auto-hyperopt class, which can be used together with strategies
+ that implement IHyperStrategy interface.
+"""
+from contextlib import suppress
+from typing import Any, Callable, Dict, List
+
+from pandas import DataFrame
+
+
+with suppress(ImportError):
+    from skopt.space import Dimension
+
+from freqtrade.optimize.hyperopt_interface import IHyperOpt
+
+
+class HyperOptAuto(IHyperOpt):
+    """
+    This class delegates functionality to Strategy(IHyperStrategy) and Strategy.HyperOpt classes.
+     Most of the time Strategy.HyperOpt class would only implement indicator_space and
+     sell_indicator_space methods, but other hyperopt methods can be overridden as well.
+    """
+
+    def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable:
+        def populate_buy_trend(dataframe: DataFrame, metadata: dict):
+            for attr_name, attr in self.strategy.enumerate_parameters('buy'):
+                if attr.optimize:
+                    # noinspection PyProtectedMember
+                    attr.value = params[attr_name]
+            return self.strategy.populate_buy_trend(dataframe, metadata)
+
+        return populate_buy_trend
+
+    def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable:
+        def populate_sell_trend(dataframe: DataFrame, metadata: dict):
+            for attr_name, attr in self.strategy.enumerate_parameters('sell'):
+                if attr.optimize:
+                    # noinspection PyProtectedMember
+                    attr.value = params[attr_name]
+            return self.strategy.populate_sell_trend(dataframe, metadata)
+
+        return populate_sell_trend
+
+    def _get_func(self, name) -> Callable:
+        """
+        Return a function defined in Strategy.HyperOpt class, or one defined in super() class.
+        :param name: function name.
+        :return: a requested function.
+        """
+        hyperopt_cls = getattr(self.strategy, 'HyperOpt', None)
+        default_func = getattr(super(), name)
+        if hyperopt_cls:
+            return getattr(hyperopt_cls, name, default_func)
+        else:
+            return default_func
+
+    def _generate_indicator_space(self, category):
+        for attr_name, attr in self.strategy.enumerate_parameters(category):
+            if attr.optimize:
+                yield attr.get_space(attr_name)
+
+    def _get_indicator_space(self, category, fallback_method_name):
+        indicator_space = list(self._generate_indicator_space(category))
+        if len(indicator_space) > 0:
+            return indicator_space
+        else:
+            return self._get_func(fallback_method_name)()
+
+    def indicator_space(self) -> List['Dimension']:
+        return self._get_indicator_space('buy', 'indicator_space')
+
+    def sell_indicator_space(self) -> List['Dimension']:
+        return self._get_indicator_space('sell', 'sell_indicator_space')
+
+    def generate_roi_table(self, params: Dict) -> Dict[int, float]:
+        return self._get_func('generate_roi_table')(params)
+
+    def roi_space(self) -> List['Dimension']:
+        return self._get_func('roi_space')()
+
+    def stoploss_space(self) -> List['Dimension']:
+        return self._get_func('stoploss_space')()
+
+    def generate_trailing_params(self, params: Dict) -> Dict:
+        return self._get_func('generate_trailing_params')(params)
+
+    def trailing_space(self) -> List['Dimension']:
+        return self._get_func('trailing_space')()

freqtrade/optimize/hyperopt_interface.py

@@ -7,11 +7,12 @@ import math
 from abc import ABC
 from typing import Any, Callable, Dict, List
 
-from skopt.space import Categorical, Dimension, Integer, Real
+from skopt.space import Categorical, Dimension, Integer
 
 from freqtrade.exceptions import OperationalException
 from freqtrade.exchange import timeframe_to_minutes
 from freqtrade.misc import round_dict
+from freqtrade.optimize.space import SKDecimal
 from freqtrade.strategy import IStrategy
 
 
@@ -31,7 +32,7 @@ class IHyperOpt(ABC):
     Defines the mandatory structure must follow any custom hyperopt
 
     Class attributes you can use:
-        ticker_interval -> int: value of the ticker interval to use for the strategy
+        timeframe -> int: value of the timeframe to use for the strategy
     """
     ticker_interval: str  # DEPRECATED
     timeframe: str
@@ -44,36 +45,31 @@ class IHyperOpt(ABC):
         IHyperOpt.ticker_interval = str(config['timeframe'])  # DEPRECATED
         IHyperOpt.timeframe = str(config['timeframe'])
 
-    @staticmethod
-    def buy_strategy_generator(params: Dict[str, Any]) -> Callable:
+    def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable:
         """
         Create a buy strategy generator.
         """
         raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy'))
 
-    @staticmethod
-    def sell_strategy_generator(params: Dict[str, Any]) -> Callable:
+    def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable:
         """
         Create a sell strategy generator.
         """
         raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell'))
 
-    @staticmethod
-    def indicator_space() -> List[Dimension]:
+    def indicator_space(self) -> List[Dimension]:
         """
         Create an indicator space.
         """
         raise OperationalException(_format_exception_message('indicator_space', 'buy'))
 
-    @staticmethod
-    def sell_indicator_space() -> List[Dimension]:
+    def sell_indicator_space(self) -> List[Dimension]:
         """
         Create a sell indicator space.
         """
         raise OperationalException(_format_exception_message('sell_indicator_space', 'sell'))
 
-    @staticmethod
-    def generate_roi_table(params: Dict) -> Dict[int, float]:
+    def generate_roi_table(self, params: Dict) -> Dict[int, float]:
         """
         Create a ROI table.
 
@@ -88,8 +84,7 @@ class IHyperOpt(ABC):
 
         return roi_table
 
-    @staticmethod
-    def roi_space() -> List[Dimension]:
+    def roi_space(self) -> List[Dimension]:
         """
         Create a ROI space.
 
@@ -97,7 +92,7 @@ class IHyperOpt(ABC):
 
         This method implements adaptive roi hyperspace with varied
         ranges for parameters which automatically adapts to the
-        ticker interval used.
+        timeframe used.
 
         It's used by Freqtrade by default, if no custom roi_space method is defined.
         """
@@ -109,7 +104,7 @@ class IHyperOpt(ABC):
         roi_t_alpha = 1.0
         roi_p_alpha = 1.0
 
-        timeframe_min = timeframe_to_minutes(IHyperOpt.ticker_interval)
+        timeframe_min = timeframe_to_minutes(self.timeframe)
 
         # We define here limits for the ROI space parameters automagically adapted to the
         # timeframe used by the bot:
@@ -119,7 +114,7 @@ class IHyperOpt(ABC):
         # * 'roi_p' (limits for the ROI value steps) components are scaled logarithmically.
         #
         # The scaling is designed so that it maps exactly to the legacy Freqtrade roi_space()
-        # method for the 5m ticker interval.
+        # method for the 5m timeframe.
         roi_t_scale = timeframe_min / 5
         roi_p_scale = math.log1p(timeframe_min) / math.log1p(5)
         roi_limits = {
@@ -145,7 +140,7 @@ class IHyperOpt(ABC):
             'roi_p2': roi_limits['roi_p2_min'],
             'roi_p3': roi_limits['roi_p3_min'],
         }
-        logger.info(f"Min roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}")
+        logger.info(f"Min roi table: {round_dict(self.generate_roi_table(p), 3)}")
         p = {
             'roi_t1': roi_limits['roi_t1_max'],
             'roi_t2': roi_limits['roi_t2_max'],
@@ -154,19 +149,21 @@ class IHyperOpt(ABC):
             'roi_p2': roi_limits['roi_p2_max'],
             'roi_p3': roi_limits['roi_p3_max'],
         }
-        logger.info(f"Max roi table: {round_dict(IHyperOpt.generate_roi_table(p), 5)}")
+        logger.info(f"Max roi table: {round_dict(self.generate_roi_table(p), 3)}")
 
         return [
             Integer(roi_limits['roi_t1_min'], roi_limits['roi_t1_max'], name='roi_t1'),
             Integer(roi_limits['roi_t2_min'], roi_limits['roi_t2_max'], name='roi_t2'),
             Integer(roi_limits['roi_t3_min'], roi_limits['roi_t3_max'], name='roi_t3'),
-            Real(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], name='roi_p1'),
-            Real(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], name='roi_p2'),
-            Real(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], name='roi_p3'),
+            SKDecimal(roi_limits['roi_p1_min'], roi_limits['roi_p1_max'], decimals=3,
+                      name='roi_p1'),
+            SKDecimal(roi_limits['roi_p2_min'], roi_limits['roi_p2_max'], decimals=3,
+                      name='roi_p2'),
+            SKDecimal(roi_limits['roi_p3_min'], roi_limits['roi_p3_max'], decimals=3,
+                      name='roi_p3'),
         ]
 
-    @staticmethod
-    def stoploss_space() -> List[Dimension]:
+    def stoploss_space(self) -> List[Dimension]:
         """
         Create a stoploss space.
 
@@ -174,11 +171,10 @@ class IHyperOpt(ABC):
         You may override it in your custom Hyperopt class.
         """
         return [
-            Real(-0.35, -0.02, name='stoploss'),
+            SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'),
         ]
 
-    @staticmethod
-    def generate_trailing_params(params: Dict) -> Dict:
+    def generate_trailing_params(self, params: Dict) -> Dict:
         """
         Create dict with trailing stop parameters.
         """
@@ -190,8 +186,7 @@ class IHyperOpt(ABC):
             'trailing_only_offset_is_reached': params['trailing_only_offset_is_reached'],
         }
 
-    @staticmethod
-    def trailing_space() -> List[Dimension]:
+    def trailing_space(self) -> List[Dimension]:
         """
         Create a trailing stoploss space.
 
@@ -206,14 +201,14 @@ class IHyperOpt(ABC):
             # other 'trailing' hyperspace parameters.
             Categorical([True], name='trailing_stop'),
 
-            Real(0.01, 0.35, name='trailing_stop_positive'),
+            SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'),
 
             # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive',
             # so this intermediate parameter is used as the value of the difference between
             # them. The value of the 'trailing_stop_positive_offset' is constructed in the
             # generate_trailing_params() method.
             # This is similar to the hyperspace dimensions used for constructing the ROI tables.
-            Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'),
+            SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'),
 
             Categorical([True, False], name='trailing_only_offset_is_reached'),
         ]

freqtrade/optimize/optimize_reports.py

@@ -110,6 +110,9 @@ def generate_pair_metrics(data: Dict[str, Dict], stake_currency: str, starting_b
 
         tabular_data.append(_generate_result_line(result, starting_balance, pair))
 
+    # Sort by total profit %:
+    tabular_data = sorted(tabular_data, key=lambda k: k['profit_total_abs'], reverse=True)
+
     # Append Total
     tabular_data.append(_generate_result_line(results, starting_balance, 'TOTAL'))
     return tabular_data

freqtrade/optimize/space/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa: F401
+from skopt.space import Categorical, Dimension, Integer, Real
+
+from .decimalspace import SKDecimal

freqtrade/optimize/space/decimalspace.py

@@ -0,0 +1,33 @@
+import numpy as np
+from skopt.space import Integer
+
+
+class SKDecimal(Integer):
+
+    def __init__(self, low, high, decimals=3, prior="uniform", base=10, transform=None,
+                 name=None, dtype=np.int64):
+        self.decimals = decimals
+        _low = int(low * pow(10, self.decimals))
+        _high = int(high * pow(10, self.decimals))
+        # trunc to precision to avoid points out of space
+        self.low_orig = round(_low * pow(0.1, self.decimals), self.decimals)
+        self.high_orig = round(_high * pow(0.1, self.decimals), self.decimals)
+
+        super().__init__(_low, _high, prior, base, transform, name, dtype)
+
+    def __repr__(self):
+        return "Decimal(low={}, high={}, decimals={}, prior='{}', transform='{}')".format(
+            self.low_orig, self.high_orig, self.decimals, self.prior, self.transform_)
+
+    def __contains__(self, point):
+        if isinstance(point, list):
+            point = np.array(point)
+        return self.low_orig <= point <= self.high_orig
+
+    def transform(self, Xt):
+        aa = [int(x * pow(10, self.decimals)) for x in Xt]
+        return super().transform(aa)
+
+    def inverse_transform(self, Xt):
+        res = super().inverse_transform(Xt)
+        return [round(x * pow(0.1, self.decimals), self.decimals) for x in res]

freqtrade/persistence/models.py

@@ -6,7 +6,6 @@ from datetime import datetime, timezone
 from decimal import Decimal
 from typing import Any, Dict, List, Optional
 
-import arrow
 from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String,
                         create_engine, desc, func, inspect)
 from sqlalchemy.exc import NoSuchModuleError
@@ -59,13 +58,10 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None:
     # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope
     # Scoped sessions proxy requests to the appropriate thread-local session.
     # We should use the scoped_session object - not a seperately initialized version
-    Trade.session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
-    Trade.query = Trade.session.query_property()
-    # Copy session attributes to order object too
-    Order.session = Trade.session
-    Order.query = Order.session.query_property()
-    PairLock.session = Trade.session
-    PairLock.query = PairLock.session.query_property()
+    Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True))
+    Trade.query = Trade._session.query_property()
+    Order.query = Trade._session.query_property()
+    PairLock.query = Trade._session.query_property()
 
     previous_tables = inspect(engine).get_table_names()
     _DECL_BASE.metadata.create_all(engine)
@@ -81,7 +77,7 @@ def cleanup_db() -> None:
     Flushes all pending operations to disk.
     :return: None
     """
-    Trade.session.flush()
+    Trade.query.session.flush()
 
 
 def clean_dry_run_db() -> None:
@@ -163,8 +159,8 @@ class Order(_DECL_BASE):
         if self.status in ('closed', 'canceled', 'cancelled'):
             self.ft_is_open = False
             if order.get('filled', 0) > 0:
-                self.order_filled_date = arrow.utcnow().datetime
-        self.order_update_date = arrow.utcnow().datetime
+                self.order_filled_date = datetime.now(timezone.utc)
+        self.order_update_date = datetime.now(timezone.utc)
 
     @staticmethod
     def update_orders(orders: List['Order'], order: Dict[str, Any]):
@@ -297,15 +293,12 @@ class LocalTrade():
             'fee_close_cost': self.fee_close_cost,
             'fee_close_currency': self.fee_close_currency,
 
-            'open_date_hum': arrow.get(self.open_date).humanize(),
             'open_date': self.open_date.strftime(DATETIME_PRINT_FORMAT),
             'open_timestamp': int(self.open_date.replace(tzinfo=timezone.utc).timestamp() * 1000),
             'open_rate': self.open_rate,
             'open_rate_requested': self.open_rate_requested,
             'open_trade_value': round(self.open_trade_value, 8),
 
-            'close_date_hum': (arrow.get(self.close_date).humanize()
-                               if self.close_date else None),
             'close_date': (self.close_date.strftime(DATETIME_PRINT_FORMAT)
                            if self.close_date else None),
             'close_timestamp': int(self.close_date.replace(
@@ -554,6 +547,8 @@ class LocalTrade():
             rate=(rate or self.close_rate),
             fee=(fee or self.fee_close)
         )
+        if self.open_trade_value == 0.0:
+            return 0.0
         profit_ratio = (close_trade_value / self.open_trade_value) - 1
         return float(f"{profit_ratio:.8f}")
 
@@ -611,7 +606,7 @@ class LocalTrade():
 
         else:
             # Not used during backtesting, but might be used by a strategy
-            sel_trades = [trade for trade in LocalTrade.trades + LocalTrade.trades_open]
+            sel_trades = list(LocalTrade.trades + LocalTrade.trades_open)
 
         if pair:
             sel_trades = [trade for trade in sel_trades if trade.pair == pair]
@@ -677,7 +672,7 @@ class LocalTrade():
         in stake currency
         """
         if Trade.use_db:
-            total_open_stake_amount = Trade.session.query(
+            total_open_stake_amount = Trade.query.with_entities(
                 func.sum(Trade.stake_amount)).filter(Trade.is_open.is_(True)).scalar()
         else:
             total_open_stake_amount = sum(
@@ -689,7 +684,7 @@ class LocalTrade():
         """
         Returns List of dicts containing all Trades, including profit and trade count
         """
-        pair_rates = Trade.session.query(
+        pair_rates = Trade.query.with_entities(
             Trade.pair,
             func.sum(Trade.close_profit).label('profit_sum'),
             func.count(Trade.pair).label('count')
@@ -712,7 +707,7 @@ class LocalTrade():
         Get best pair with closed trade.
         :returns: Tuple containing (pair, profit_sum)
         """
-        best_pair = Trade.session.query(
+        best_pair = Trade.query.with_entities(
             Trade.pair, func.sum(Trade.close_profit).label('profit_sum')
         ).filter(Trade.is_open.is_(False)) \
             .group_by(Trade.pair) \
@@ -805,10 +800,10 @@ class Trade(_DECL_BASE, LocalTrade):
     def delete(self) -> None:
 
         for order in self.orders:
-            Order.session.delete(order)
+            Order.query.session.delete(order)
 
-        Trade.session.delete(self)
-        Trade.session.flush()
+        Trade.query.session.delete(self)
+        Trade.query.session.flush()
 
     @staticmethod
     def get_trades_proxy(*, pair: str = None, is_open: bool = None,

freqtrade/persistence/pairlock_middleware.py

@@ -48,8 +48,8 @@ class PairLocks():
             active=True
         )
         if PairLocks.use_db:
-            PairLock.session.add(lock)
-            PairLock.session.flush()
+            PairLock.query.session.add(lock)
+            PairLock.query.session.flush()
         else:
             PairLocks.locks.append(lock)
 
@@ -99,7 +99,7 @@ class PairLocks():
         for lock in locks:
             lock.active = False
         if PairLocks.use_db:
-            PairLock.session.flush()
+            PairLock.query.session.flush()
 
     @staticmethod
     def is_global_lock(now: Optional[datetime] = None) -> bool:

freqtrade/plot/plotting.py

@@ -441,7 +441,7 @@ def generate_candlestick_graph(pair: str, data: pd.DataFrame, trades: pd.DataFra
 
 
 def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
-                          trades: pd.DataFrame, timeframe: str) -> go.Figure:
+                          trades: pd.DataFrame, timeframe: str, stake_currency: str) -> go.Figure:
     # Combine close-values for all pairs, rename columns to "pair"
     df_comb = combine_dataframes_with_mean(data, "close")
 
@@ -466,8 +466,8 @@ def generate_profit_graph(pairs: str, data: Dict[str, pd.DataFrame],
                         subplot_titles=["AVG Close Price", "Combined Profit", "Profit per pair"])
     fig['layout'].update(title="Freqtrade Profit plot")
     fig['layout']['yaxis1'].update(title='Price')
-    fig['layout']['yaxis2'].update(title='Profit')
-    fig['layout']['yaxis3'].update(title='Profit')
+    fig['layout']['yaxis2'].update(title=f'Profit {stake_currency}')
+    fig['layout']['yaxis3'].update(title=f'Profit {stake_currency}')
     fig['layout']['xaxis']['rangeslider'].update(visible=False)
 
     fig.add_trace(avgclose, 1, 1)
@@ -581,6 +581,7 @@ def plot_profit(config: Dict[str, Any]) -> None:
     # Create an average close price of all the pairs that were involved.
     # this could be useful to gauge the overall market trend
     fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'],
-                                trades, config.get('timeframe', '5m'))
+                                trades, config.get('timeframe', '5m'),
+                                config.get('stake_currency', ''))
     store_plot_file(fig, filename='freqtrade-profit-plot.html',
                     directory=config['user_data_dir'] / 'plot', auto_open=True)

freqtrade/plugins/pairlist/IPairList.py

@@ -73,7 +73,7 @@ class IPairList(LoggingMixin, ABC):
         """
         raise NotImplementedError()
 
-    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+    def gen_pairlist(self, tickers: Dict) -> List[str]:
         """
         Generate the pairlist.
 
@@ -84,8 +84,7 @@ class IPairList(LoggingMixin, ABC):
         it will raise the exception if a Pairlist Handler is used at the first
         position in the chain.
 
-        :param cached_pairlist: Previously generated pairlist (cached)
-        :param tickers: Tickers (from exchange.get_tickers()).
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: List of pairs
         """
         raise OperationalException("This Pairlist Handler should not be used "

freqtrade/plugins/pairlist/PerformanceFilter.py

@@ -2,7 +2,7 @@
 Performance pair list filter
 """
 import logging
-from typing import Any, Dict, List
+from typing import Dict, List
 
 import pandas as pd
 
@@ -15,11 +15,6 @@ logger = logging.getLogger(__name__)
 
 class PerformanceFilter(IPairList):
 
-    def __init__(self, exchange, pairlistmanager,
-                 config: Dict[str, Any], pairlistconfig: Dict[str, Any],
-                 pairlist_pos: int) -> None:
-        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
-
     @property
     def needstickers(self) -> bool:
         """

freqtrade/plugins/pairlist/StaticPairList.py

@@ -42,11 +42,10 @@ class StaticPairList(IPairList):
         """
         return f"{self.name}"
 
-    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+    def gen_pairlist(self, tickers: Dict) -> List[str]:
         """
         Generate the pairlist
-        :param cached_pairlist: Previously generated pairlist (cached)
-        :param tickers: Tickers (from exchange.get_tickers()).
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: List of pairs
         """
         if self._allow_inactive:

freqtrade/plugins/pairlist/VolatilityFilter.py

@@ -0,0 +1,121 @@
+"""
+Volatility pairlist filter
+"""
+import logging
+import sys
+from copy import deepcopy
+from typing import Any, Dict, List, Optional
+
+import arrow
+import numpy as np
+from cachetools.ttl import TTLCache
+from pandas import DataFrame
+
+from freqtrade.exceptions import OperationalException
+from freqtrade.misc import plural
+from freqtrade.plugins.pairlist.IPairList import IPairList
+
+
+logger = logging.getLogger(__name__)
+
+
+class VolatilityFilter(IPairList):
+    '''
+    Filters pairs by volatility
+    '''
+
+    def __init__(self, exchange, pairlistmanager,
+                 config: Dict[str, Any], pairlistconfig: Dict[str, Any],
+                 pairlist_pos: int) -> None:
+        super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos)
+
+        self._days = pairlistconfig.get('lookback_days', 10)
+        self._min_volatility = pairlistconfig.get('min_volatility', 0)
+        self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize)
+        self._refresh_period = pairlistconfig.get('refresh_period', 1440)
+
+        self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period)
+
+        if self._days < 1:
+            raise OperationalException("VolatilityFilter requires lookback_days to be >= 1")
+        if self._days > exchange.ohlcv_candle_limit('1d'):
+            raise OperationalException("VolatilityFilter requires lookback_days to not "
+                                       "exceed exchange max request size "
+                                       f"({exchange.ohlcv_candle_limit('1d')})")
+
+    @property
+    def needstickers(self) -> bool:
+        """
+        Boolean property defining if tickers are necessary.
+        If no Pairlist requires tickers, an empty List is passed
+        as tickers argument to filter_pairlist
+        """
+        return False
+
+    def short_desc(self) -> str:
+        """
+        Short whitelist method description - used for startup-messages
+        """
+        return (f"{self.name} - Filtering pairs with volatility range "
+                f"{self._min_volatility}-{self._max_volatility} "
+                f" the last {self._days} {plural(self._days, 'day')}.")
+
+    def filter_pairlist(self, pairlist: List[str], tickers: Dict) -> List[str]:
+        """
+        Validate trading range
+        :param pairlist: pairlist to filter or sort
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
+        :return: new allowlist
+        """
+        needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache]
+
+        since_ms = int(arrow.utcnow()
+                       .floor('day')
+                       .shift(days=-self._days - 1)
+                       .float_timestamp) * 1000
+        # Get all candles
+        candles = {}
+        if needed_pairs:
+            candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms,
+                                                          cache=False)
+
+        if self._enabled:
+            for p in deepcopy(pairlist):
+                daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None
+                if not self._validate_pair_loc(p, daily_candles):
+                    pairlist.remove(p)
+        return pairlist
+
+    def _validate_pair_loc(self, pair: str, daily_candles: Optional[DataFrame]) -> bool:
+        """
+        Validate trading range
+        :param pair: Pair that's currently validated
+        :param ticker: ticker dict as returned from ccxt.load_markets()
+        :return: True if the pair can stay, false if it should be removed
+        """
+        # Check symbol in cache
+        cached_res = self._pair_cache.get(pair, None)
+        if cached_res is not None:
+            return cached_res
+
+        result = False
+        if daily_candles is not None and not daily_candles.empty:
+            returns = (np.log(daily_candles.close / daily_candles.close.shift(-1)))
+            returns.fillna(0, inplace=True)
+
+            volatility_series = returns.rolling(window=self._days).std()*np.sqrt(self._days)
+            volatility_avg = volatility_series.mean()
+
+            if self._min_volatility <= volatility_avg <= self._max_volatility:
+                result = True
+            else:
+                self.log_once(f"Removed {pair} from whitelist, because volatility "
+                              f"over {self._days} {plural(self._days, 'day')} "
+                              f"is: {volatility_avg:.3f} "
+                              f"which is not in the configured range of "
+                              f"{self._min_volatility}-{self._max_volatility}.",
+                              logger.info)
+                result = False
+            self._pair_cache[pair] = result
+
+        return result

freqtrade/plugins/pairlist/VolumePairList.py

@@ -4,9 +4,10 @@ Volume PairList provider
 Provides dynamic pair list based on trade volumes
 """
 import logging
-from datetime import datetime
 from typing import Any, Dict, List
 
+from cachetools.ttl import TTLCache
+
 from freqtrade.exceptions import OperationalException
 from freqtrade.plugins.pairlist.IPairList import IPairList
 
@@ -33,7 +34,8 @@ class VolumePairList(IPairList):
         self._number_pairs = self._pairlistconfig['number_assets']
         self._sort_key = self._pairlistconfig.get('sort_key', 'quoteVolume')
         self._min_value = self._pairlistconfig.get('min_value', 0)
-        self.refresh_period = self._pairlistconfig.get('refresh_period', 1800)
+        self._refresh_period = self._pairlistconfig.get('refresh_period', 1800)
+        self._pair_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
 
         if not self._exchange.exchange_has('fetchTickers'):
             raise OperationalException(
@@ -63,17 +65,19 @@ class VolumePairList(IPairList):
         """
         return f"{self.name} - top {self._pairlistconfig['number_assets']} volume pairs."
 
-    def gen_pairlist(self, cached_pairlist: List[str], tickers: Dict) -> List[str]:
+    def gen_pairlist(self, tickers: Dict) -> List[str]:
         """
         Generate the pairlist
-        :param cached_pairlist: Previously generated pairlist (cached)
-        :param tickers: Tickers (from exchange.get_tickers()).
+        :param tickers: Tickers (from exchange.get_tickers()). May be cached.
         :return: List of pairs
         """
         # Generate dynamic whitelist
         # Must always run if this pairlist is not the first in the list.
-        if self._last_refresh + self.refresh_period < datetime.now().timestamp():
-            self._last_refresh = int(datetime.now().timestamp())
+        pairlist = self._pair_cache.get('pairlist')
+        if pairlist:
+            # Item found - no refresh necessary
+            return pairlist
+        else:
 
             # Use fresh pairlist
             # Check if pair quote currency equals to the stake currency.
@@ -82,9 +86,9 @@ class VolumePairList(IPairList):
                     if (self._exchange.get_pair_quote_currency(k) == self._stake_currency
                         and v[self._sort_key] is not None)]
             pairlist = [s['symbol'] for s in filtered_tickers]
-        else:
-            # Use the cached pairlist if it's not time yet to refresh
-            pairlist = cached_pairlist
+
+            pairlist = self.filter_pairlist(pairlist, tickers)
+            self._pair_cache['pairlist'] = pairlist
 
         return pairlist
 

freqtrade/plugins/pairlist/rangestabilityfilter.py

@@ -87,8 +87,9 @@ class RangeStabilityFilter(IPairList):
         :return: True if the pair can stay, false if it should be removed
         """
         # Check symbol in cache
-        if pair in self._pair_cache:
-            return self._pair_cache[pair]
+        cached_res = self._pair_cache.get(pair, None)
+        if cached_res is not None:
+            return cached_res
 
         result = False
         if daily_candles is not None and not daily_candles.empty:

freqtrade/plugins/pairlistmanager.py

@@ -3,7 +3,7 @@ PairList manager class
 """
 import logging
 from copy import deepcopy
-from typing import Any, Dict, List
+from typing import Dict, List
 
 from cachetools import TTLCache, cached
 
@@ -79,11 +79,8 @@ class PairListManager():
         if self._tickers_needed:
             tickers = self._get_cached_tickers()
 
-        # Adjust whitelist if filters are using tickers
-        pairlist = self._prepare_whitelist(self._whitelist.copy(), tickers)
-
         # Generate the pairlist with first Pairlist Handler in the chain
-        pairlist = self._pairlist_handlers[0].gen_pairlist(self._whitelist, tickers)
+        pairlist = self._pairlist_handlers[0].gen_pairlist(tickers)
 
         # Process all Pairlist Handlers in the chain
         for pairlist_handler in self._pairlist_handlers:
@@ -95,19 +92,6 @@ class PairListManager():
 
         self._whitelist = pairlist
 
-    def _prepare_whitelist(self, pairlist: List[str], tickers: Dict[str, Any]) -> List[str]:
-        """
-        Prepare sanitized pairlist for Pairlist Handlers that use tickers data - remove
-        pairs that do not have ticker available
-        """
-        if self._tickers_needed:
-            # Copy list since we're modifying this list
-            for p in deepcopy(pairlist):
-                if p not in tickers:
-                    pairlist.remove(p)
-
-        return pairlist
-
     def verify_blacklist(self, pairlist: List[str], logmethod) -> List[str]:
         """
         Verify and remove items from pairlist - returning a filtered pairlist.

freqtrade/plugins/protections/cooldown_period.py

@@ -1,7 +1,6 @@
 
 import logging
 from datetime import datetime, timedelta
-from typing import Any, Dict
 
 from freqtrade.persistence import Trade
 from freqtrade.plugins.protections import IProtection, ProtectionReturn
@@ -15,9 +14,6 @@ class CooldownPeriod(IProtection):
     has_global_stop: bool = False
     has_local_stop: bool = True
 
-    def __init__(self, config: Dict[str, Any], protection_config: Dict[str, Any]) -> None:
-        super().__init__(config, protection_config)
-
     def _reason(self) -> str:
         """
         LockReason to use

freqtrade/plugins/protections/max_drawdown_protection.py

@@ -61,7 +61,7 @@ class MaxDrawdown(IProtection):
 
         if drawdown > self._max_allowed_drawdown:
             self.log_once(
-                f"Trading stopped due to Max Drawdown {drawdown:.2f} < {self._max_allowed_drawdown}"
+                f"Trading stopped due to Max Drawdown {drawdown:.2f} > {self._max_allowed_drawdown}"
                 f" within {self.lookback_period_str}.", logger.info)
             until = self.calculate_lock_end(trades, self._stop_duration)
 

freqtrade/resolvers/iresolver.py

@@ -61,7 +61,7 @@ class IResolver:
         module = importlib.util.module_from_spec(spec)
         try:
             spec.loader.exec_module(module)  # type: ignore # importlib does not use typehints
-        except (ModuleNotFoundError, SyntaxError, ImportError) as err:
+        except (ModuleNotFoundError, SyntaxError, ImportError, NameError) as err:
             # Catch errors in case a specific module is not installed
             logger.warning(f"Could not import {module_path} due to '{err}'")
             if enum_failed:

freqtrade/resolvers/strategy_resolver.py

@@ -196,9 +196,9 @@ class StrategyResolver(IResolver):
             strategy._populate_fun_len = len(getfullargspec(strategy.populate_indicators).args)
             strategy._buy_fun_len = len(getfullargspec(strategy.populate_buy_trend).args)
             strategy._sell_fun_len = len(getfullargspec(strategy.populate_sell_trend).args)
-            if any([x == 2 for x in [strategy._populate_fun_len,
-                                     strategy._buy_fun_len,
-                                     strategy._sell_fun_len]]):
+            if any(x == 2 for x in [strategy._populate_fun_len,
+                                    strategy._buy_fun_len,
+                                    strategy._sell_fun_len]):
                 strategy.INTERFACE_VERSION = 1
 
             return strategy

freqtrade/rpc/api_server/api_schemas.py

@@ -151,13 +151,11 @@ class TradeSchema(BaseModel):
     fee_close: Optional[float]
     fee_close_cost: Optional[float]
     fee_close_currency: Optional[str]
-    open_date_hum: str
     open_date: str
     open_timestamp: int
     open_rate: float
     open_rate_requested: Optional[float]
     open_trade_value: float
-    close_date_hum: Optional[str]
     close_date: Optional[str]
     close_timestamp: Optional[int]
     close_rate: Optional[float]
@@ -168,6 +166,7 @@ class TradeSchema(BaseModel):
     profit_ratio: Optional[float]
     profit_pct: Optional[float]
     profit_abs: Optional[float]
+    profit_fiat: Optional[float]
     sell_reason: Optional[str]
     sell_order_status: Optional[str]
     stop_loss_abs: Optional[float]
@@ -190,7 +189,6 @@ class OpenTradeSchema(TradeSchema):
     stoploss_current_dist_ratio: Optional[float]
     stoploss_entry_dist: Optional[float]
     stoploss_entry_dist_ratio: Optional[float]
-    base_currency: str
     current_profit: float
     current_profit_abs: float
     current_profit_pct: float
@@ -201,6 +199,7 @@ class OpenTradeSchema(TradeSchema):
 class TradeResponse(BaseModel):
     trades: List[TradeSchema]
     trades_count: int
+    total_trades: int
 
 
 class ForceBuyResponse(BaseModel):

freqtrade/rpc/api_server/api_v1.py

@@ -17,8 +17,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac
                                                   OpenTradeSchema, PairHistory, PerformanceEntry,
                                                   Ping, PlotConfig, Profit, ResultMsg, ShowConfig,
                                                   Stats, StatusMsg, StrategyListResponse,
-                                                  StrategyResponse, TradeResponse, Version,
-                                                  WhitelistResponse)
+                                                  StrategyResponse, Version, WhitelistResponse)
 from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional
 from freqtrade.rpc.rpc import RPCException
 
@@ -83,9 +82,19 @@ def status(rpc: RPC = Depends(get_rpc)):
         return []
 
 
-@router.get('/trades', response_model=TradeResponse, tags=['info', 'trading'])
-def trades(limit: int = 0, rpc: RPC = Depends(get_rpc)):
-    return rpc._rpc_trade_history(limit)
+# Using the responsemodel here will cause a ~100% increase in response time (from 1s to 2s)
+# on big databases. Correct response model: response_model=TradeResponse,
+@router.get('/trades', tags=['info', 'trading'])
+def trades(limit: int = 500, offset: int = 0, rpc: RPC = Depends(get_rpc)):
+    return rpc._rpc_trade_history(limit, offset=offset, order_by_id=True)
+
+
+@router.get('/trade/{tradeid}', response_model=OpenTradeSchema, tags=['info', 'trading'])
+def trade(tradeid: int = 0, rpc: RPC = Depends(get_rpc)):
+    try:
+        return rpc._rpc_trade_status([tradeid])[0]
+    except (RPCException, KeyError):
+        raise HTTPException(status_code=404, detail='Trade not found.')
 
 
 @router.delete('/trades/{tradeid}', response_model=DeleteTrade, tags=['info', 'trading'])

freqtrade/rpc/api_server/web_ui.py

@@ -13,6 +13,11 @@ async def favicon():
     return FileResponse(str(Path(__file__).parent / 'ui/favicon.ico'))
 
 
+@router_ui.get('/fallback_file.html', include_in_schema=False)
+async def fallback():
+    return FileResponse(str(Path(__file__).parent / 'ui/fallback_file.html'))
+
+
 @router_ui.get('/{rest_of_path:path}', include_in_schema=False)
 async def index_html(rest_of_path: str):
     """

freqtrade/rpc/fiat_convert.py

@@ -4,9 +4,9 @@ e.g BTC to USD
 """
 
 import logging
-import time
-from typing import Dict, List
+from typing import Dict
 
+from cachetools.ttl import TTLCache
 from pycoingecko import CoinGeckoAPI
 
 from freqtrade.constants import SUPPORTED_FIAT
@@ -15,51 +15,6 @@ from freqtrade.constants import SUPPORTED_FIAT
 logger = logging.getLogger(__name__)
 
 
-class CryptoFiat:
-    """
-    Object to describe what is the price of Crypto-currency in a FIAT
-    """
-    # Constants
-    CACHE_DURATION = 6 * 60 * 60  # 6 hours
-
-    def __init__(self, crypto_symbol: str, fiat_symbol: str, price: float) -> None:
-        """
-        Create an object that will contains the price for a crypto-currency in fiat
-        :param crypto_symbol: Crypto-currency you want to convert (e.g BTC)
-        :param fiat_symbol: FIAT currency you want to convert to (e.g USD)
-        :param price: Price in FIAT
-        """
-
-        # Public attributes
-        self.crypto_symbol = None
-        self.fiat_symbol = None
-        self.price = 0.0
-
-        # Private attributes
-        self._expiration = 0.0
-
-        self.crypto_symbol = crypto_symbol.lower()
-        self.fiat_symbol = fiat_symbol.lower()
-        self.set_price(price=price)
-
-    def set_price(self, price: float) -> None:
-        """
-        Set the price of the Crypto-currency in FIAT and set the expiration time
-        :param price: Price of the current Crypto currency in the fiat
-        :return: None
-        """
-        self.price = price
-        self._expiration = time.time() + self.CACHE_DURATION
-
-    def is_expired(self) -> bool:
-        """
-        Return if the current price is still valid or needs to be refreshed
-        :return: bool, true the price is expired and needs to be refreshed, false the price is
-         still valid
-        """
-        return self._expiration - time.time() <= 0
-
-
 class CryptoToFiatConverter:
     """
     Main class to initiate Crypto to FIAT.
@@ -84,7 +39,9 @@ class CryptoToFiatConverter:
         return CryptoToFiatConverter.__instance
 
     def __init__(self) -> None:
-        self._pairs: List[CryptoFiat] = []
+        # Timeout: 6h
+        self._pair_price: TTLCache = TTLCache(maxsize=500, ttl=6 * 60 * 60)
+
         self._load_cryptomap()
 
     def _load_cryptomap(self) -> None:
@@ -118,49 +75,31 @@ class CryptoToFiatConverter:
         """
         crypto_symbol = crypto_symbol.lower()
         fiat_symbol = fiat_symbol.lower()
+        inverse = False
 
+        if crypto_symbol == 'usd':
+            # usd corresponds to "uniswap-state-dollar" for coingecko.
+            # We'll therefore need to "swap" the currencies
+            logger.info(f"reversing Rates {crypto_symbol}, {fiat_symbol}")
+            crypto_symbol = fiat_symbol
+            fiat_symbol = 'usd'
+            inverse = True
+
+        symbol = f"{crypto_symbol}/{fiat_symbol}"
         # Check if the fiat convertion you want is supported
         if not self._is_supported_fiat(fiat=fiat_symbol):
             raise ValueError(f'The fiat {fiat_symbol} is not supported.')
 
-        # Get the pair that interest us and return the price in fiat
-        for pair in self._pairs:
-            if pair.crypto_symbol == crypto_symbol and pair.fiat_symbol == fiat_symbol:
-                # If the price is expired we refresh it, avoid to call the API all the time
-                if pair.is_expired():
-                    pair.set_price(
-                        price=self._find_price(
-                            crypto_symbol=pair.crypto_symbol,
-                            fiat_symbol=pair.fiat_symbol
-                        )
-                    )
+        price = self._pair_price.get(symbol, None)
 
-                # return the last price we have for this pair
-                return pair.price
-
-        # The pair does not exist, so we create it and return the price
-        return self._add_pair(
-            crypto_symbol=crypto_symbol,
-            fiat_symbol=fiat_symbol,
-            price=self._find_price(
+        if not price:
+            price = self._find_price(
                 crypto_symbol=crypto_symbol,
                 fiat_symbol=fiat_symbol
             )
-        )
-
-    def _add_pair(self, crypto_symbol: str, fiat_symbol: str, price: float) -> float:
-        """
-        :param crypto_symbol: Crypto-currency you want to convert (e.g BTC)
-        :param fiat_symbol: FIAT currency you want to convert to (e.g USD)
-        :return: price in FIAT
-        """
-        self._pairs.append(
-            CryptoFiat(
-                crypto_symbol=crypto_symbol,
-                fiat_symbol=fiat_symbol,
-                price=price
-            )
-        )
+            if inverse and price != 0.0:
+                price = 1 / price
+            self._pair_price[symbol] = price
 
         return price
 

freqtrade/rpc/rpc.py

@@ -31,13 +31,15 @@ logger = logging.getLogger(__name__)
 
 
 class RPCMessageType(Enum):
-    STATUS_NOTIFICATION = 'status'
-    WARNING_NOTIFICATION = 'warning'
-    STARTUP_NOTIFICATION = 'startup'
-    BUY_NOTIFICATION = 'buy'
-    BUY_CANCEL_NOTIFICATION = 'buy_cancel'
-    SELL_NOTIFICATION = 'sell'
-    SELL_CANCEL_NOTIFICATION = 'sell_cancel'
+    STATUS = 'status'
+    WARNING = 'warning'
+    STARTUP = 'startup'
+    BUY = 'buy'
+    BUY_FILL = 'buy_fill'
+    BUY_CANCEL = 'buy_cancel'
+    SELL = 'sell'
+    SELL_FILL = 'sell_fill'
+    SELL_CANCEL = 'sell_cancel'
 
     def __repr__(self):
         return self.value
@@ -167,12 +169,24 @@ class RPC:
                 if trade.open_order_id:
                     order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair)
                 # calculate profit and send message to user
-                try:
-                    current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
-                except (ExchangeError, PricingError):
-                    current_rate = NAN
+                if trade.is_open:
+                    try:
+                        current_rate = self._freqtrade.get_sell_rate(trade.pair, False)
+                    except (ExchangeError, PricingError):
+                        current_rate = NAN
+                else:
+                    current_rate = trade.close_rate
                 current_profit = trade.calc_profit_ratio(current_rate)
                 current_profit_abs = trade.calc_profit(current_rate)
+
+                # Calculate fiat profit
+                if self._fiat_converter:
+                    current_profit_fiat = self._fiat_converter.convert_amount(
+                        current_profit_abs,
+                        self._freqtrade.config['stake_currency'],
+                        self._freqtrade.config['fiat_display_currency']
+                    )
+
                 # Calculate guaranteed profit (in case of trailing stop)
                 stoploss_entry_dist = trade.calc_profit(trade.stop_loss)
                 stoploss_entry_dist_ratio = trade.calc_profit_ratio(trade.stop_loss)
@@ -191,6 +205,7 @@ class RPC:
                     profit_ratio=current_profit,
                     profit_pct=round(current_profit * 100, 2),
                     profit_abs=current_profit_abs,
+                    profit_fiat=current_profit_fiat,
 
                     stoploss_current_dist=stoploss_current_dist,
                     stoploss_current_dist_ratio=round(stoploss_current_dist_ratio, 8),
@@ -285,11 +300,12 @@ class RPC:
             'data': data
         }
 
-    def _rpc_trade_history(self, limit: int) -> Dict:
+    def _rpc_trade_history(self, limit: int, offset: int = 0, order_by_id: bool = False) -> Dict:
         """ Returns the X last trades """
-        if limit > 0:
+        order_by = Trade.id if order_by_id else Trade.close_date.desc()
+        if limit:
             trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
-                Trade.close_date.desc()).limit(limit)
+                order_by).limit(limit).offset(offset)
         else:
             trades = Trade.get_trades([Trade.is_open.is_(False)]).order_by(
                 Trade.close_date.desc()).all()
@@ -298,7 +314,8 @@ class RPC:
 
         return {
             "trades": output,
-            "trades_count": len(output)
+            "trades_count": len(output),
+            "total_trades": Trade.get_trades([Trade.is_open.is_(False)]).count(),
         }
 
     def _rpc_stats(self) -> Dict[str, Any]:
@@ -432,7 +449,7 @@ class RPC:
         output = []
         total = 0.0
         try:
-            tickers = self._freqtrade.exchange.get_tickers()
+            tickers = self._freqtrade.exchange.get_tickers(cached=True)
         except (ExchangeError):
             raise RPCException('Error getting current tickers.')
 
@@ -548,7 +565,7 @@ class RPC:
                 # Execute sell for all open orders
                 for trade in Trade.get_open_trades():
                     _exec_forcesell(trade)
-                Trade.session.flush()
+                Trade.query.session.flush()
                 self._freqtrade.wallets.update()
                 return {'result': 'Created sell orders for all open trades.'}
 
@@ -561,7 +578,7 @@ class RPC:
                 raise RPCException('invalid argument')
 
             _exec_forcesell(trade)
-            Trade.session.flush()
+            Trade.query.session.flush()
             self._freqtrade.wallets.update()
             return {'result': f'Created sell order for trade {trade_id}.'}
 
@@ -590,8 +607,7 @@ class RPC:
             raise RPCException(f'position for {pair} already open - id: {trade.id}')
 
         # gen stake amount
-        stakeamount = self._freqtrade.wallets.get_trade_stake_amount(
-            pair, self._freqtrade.get_free_open_trades())
+        stakeamount = self._freqtrade.wallets.get_trade_stake_amount(pair)
 
         # execute buy
         if self._freqtrade.execute_buy(pair, stakeamount, price, forcebuy=True):
@@ -686,7 +702,7 @@ class RPC:
             lock.lock_end_time = datetime.now(timezone.utc)
 
         # session is always the same
-        PairLock.session.flush()
+        PairLock.query.session.flush()
 
         return self._rpc_locks()
 

freqtrade/rpc/rpc_manager.py

@@ -67,7 +67,7 @@ class RPCManager:
     def startup_messages(self, config: Dict[str, Any], pairlist, protections) -> None:
         if config['dry_run']:
             self.send_msg({
-                'type': RPCMessageType.WARNING_NOTIFICATION,
+                'type': RPCMessageType.WARNING,
                 'status': 'Dry run is enabled. All trades are simulated.'
             })
         stake_currency = config['stake_currency']
@@ -79,7 +79,7 @@ class RPCManager:
         exchange_name = config['exchange']['name']
         strategy_name = config.get('strategy', '')
         self.send_msg({
-            'type': RPCMessageType.STARTUP_NOTIFICATION,
+            'type': RPCMessageType.STARTUP,
             'status': f'*Exchange:* `{exchange_name}`\n'
                       f'*Stake per trade:* `{stake_amount} {stake_currency}`\n'
                       f'*Minimum ROI:* `{minimal_roi}`\n'
@@ -88,13 +88,13 @@ class RPCManager:
                       f'*Strategy:* `{strategy_name}`'
         })
         self.send_msg({
-            'type': RPCMessageType.STARTUP_NOTIFICATION,
+            'type': RPCMessageType.STARTUP,
             'status': f'Searching for {stake_currency} pairs to buy and sell '
                       f'based on {pairlist.short_desc()}'
         })
         if len(protections.name_list) > 0:
             prots = '\n'.join([p for prot in protections.short_desc() for k, p in prot.items()])
             self.send_msg({
-                'type': RPCMessageType.STARTUP_NOTIFICATION,
+                'type': RPCMessageType.STARTUP,
                 'status': f'Using Protections: \n{prots}'
             })

freqtrade/rpc/telegram.py

@@ -159,10 +159,10 @@ class Telegram(RPCHandler):
         for handle in handles:
             self._updater.dispatcher.add_handler(handle)
         self._updater.start_polling(
-            clean=True,
             bootstrap_retries=-1,
             timeout=30,
             read_latency=60,
+            drop_pending_updates=True,
         )
         logger.info(
             'rpc.telegram is listening for following commands: %s',
@@ -176,6 +176,53 @@ class Telegram(RPCHandler):
         """
         self._updater.stop()
 
+    def _format_buy_msg(self, msg: Dict[str, Any]) -> str:
+        if self._rpc._fiat_converter:
+            msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount(
+                msg['stake_amount'], msg['stake_currency'], msg['fiat_currency'])
+        else:
+            msg['stake_amount_fiat'] = 0
+
+        message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}"
+                   f" (#{msg['trade_id']})\n"
+                   f"*Amount:* `{msg['amount']:.8f}`\n"
+                   f"*Open Rate:* `{msg['limit']:.8f}`\n"
+                   f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
+                   f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}")
+
+        if msg.get('fiat_currency', None):
+            message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
+        message += ")`"
+        return message
+
+    def _format_sell_msg(self, msg: Dict[str, Any]) -> str:
+        msg['amount'] = round(msg['amount'], 8)
+        msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2)
+        msg['duration'] = msg['close_date'].replace(
+            microsecond=0) - msg['open_date'].replace(microsecond=0)
+        msg['duration_min'] = msg['duration'].total_seconds() / 60
+
+        msg['emoji'] = self._get_sell_emoji(msg)
+
+        message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n"
+                   "*Amount:* `{amount:.8f}`\n"
+                   "*Open Rate:* `{open_rate:.8f}`\n"
+                   "*Current Rate:* `{current_rate:.8f}`\n"
+                   "*Close Rate:* `{limit:.8f}`\n"
+                   "*Sell Reason:* `{sell_reason}`\n"
+                   "*Duration:* `{duration} ({duration_min:.1f} min)`\n"
+                   "*Profit:* `{profit_percent:.2f}%`").format(**msg)
+
+        # Check if all sell properties are available.
+        # This might not be the case if the message origin is triggered by /forcesell
+        if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency'])
+                and self._rpc._fiat_converter):
+            msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount(
+                msg['profit_amount'], msg['stake_currency'], msg['fiat_currency'])
+            message += (' `({gain}: {profit_amount:.8f} {stake_currency}'
+                        ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg)
+        return message
+
     def send_msg(self, msg: Dict[str, Any]) -> None:
         """ Send a message to telegram channel """
 
@@ -186,67 +233,33 @@ class Telegram(RPCHandler):
             # Notification disabled
             return
 
-        if msg['type'] == RPCMessageType.BUY_NOTIFICATION:
-            if self._rpc._fiat_converter:
-                msg['stake_amount_fiat'] = self._rpc._fiat_converter.convert_amount(
-                    msg['stake_amount'], msg['stake_currency'], msg['fiat_currency'])
-            else:
-                msg['stake_amount_fiat'] = 0
+        if msg['type'] == RPCMessageType.BUY:
+            message = self._format_buy_msg(msg)
 
-            message = (f"\N{LARGE BLUE CIRCLE} *{msg['exchange']}:* Buying {msg['pair']}"
-                       f" (#{msg['trade_id']})\n"
-                       f"*Amount:* `{msg['amount']:.8f}`\n"
-                       f"*Open Rate:* `{msg['limit']:.8f}`\n"
-                       f"*Current Rate:* `{msg['current_rate']:.8f}`\n"
-                       f"*Total:* `({round_coin_value(msg['stake_amount'], msg['stake_currency'])}")
-
-            if msg.get('fiat_currency', None):
-                message += f", {round_coin_value(msg['stake_amount_fiat'], msg['fiat_currency'])}"
-            message += ")`"
-
-        elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION:
+        elif msg['type'] in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL):
+            msg['message_side'] = 'buy' if msg['type'] == RPCMessageType.BUY_CANCEL else 'sell'
             message = ("\N{WARNING SIGN} *{exchange}:* "
-                       "Cancelling open buy Order for {pair} (#{trade_id}). "
+                       "Cancelling open {message_side} Order for {pair} (#{trade_id}). "
                        "Reason: {reason}.".format(**msg))
 
-        elif msg['type'] == RPCMessageType.SELL_NOTIFICATION:
-            msg['amount'] = round(msg['amount'], 8)
-            msg['profit_percent'] = round(msg['profit_ratio'] * 100, 2)
-            msg['duration'] = msg['close_date'].replace(
-                microsecond=0) - msg['open_date'].replace(microsecond=0)
-            msg['duration_min'] = msg['duration'].total_seconds() / 60
+        elif msg['type'] == RPCMessageType.BUY_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Buy order for {pair} (#{trade_id}) filled "
+                       "for {open_rate}.".format(**msg))
+        elif msg['type'] == RPCMessageType.SELL_FILL:
+            message = ("\N{LARGE CIRCLE} *{exchange}:* "
+                       "Sell order for {pair} (#{trade_id}) filled "
+                       "for {close_rate}.".format(**msg))
+        elif msg['type'] == RPCMessageType.SELL:
+            message = self._format_sell_msg(msg)
 
-            msg['emoji'] = self._get_sell_emoji(msg)
-
-            message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n"
-                       "*Amount:* `{amount:.8f}`\n"
-                       "*Open Rate:* `{open_rate:.8f}`\n"
-                       "*Current Rate:* `{current_rate:.8f}`\n"
-                       "*Close Rate:* `{limit:.8f}`\n"
-                       "*Sell Reason:* `{sell_reason}`\n"
-                       "*Duration:* `{duration} ({duration_min:.1f} min)`\n"
-                       "*Profit:* `{profit_percent:.2f}%`").format(**msg)
-
-            # Check if all sell properties are available.
-            # This might not be the case if the message origin is triggered by /forcesell
-            if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency'])
-                    and self._rpc._fiat_converter):
-                msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount(
-                    msg['profit_amount'], msg['stake_currency'], msg['fiat_currency'])
-                message += (' `({gain}: {profit_amount:.8f} {stake_currency}'
-                            ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg)
-
-        elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
-            message = ("\N{WARNING SIGN} *{exchange}:* Cancelling Open Sell Order "
-                       "for {pair} (#{trade_id}). Reason: {reason}").format(**msg)
-
-        elif msg['type'] == RPCMessageType.STATUS_NOTIFICATION:
+        elif msg['type'] == RPCMessageType.STATUS:
             message = '*Status:* `{status}`'.format(**msg)
 
-        elif msg['type'] == RPCMessageType.WARNING_NOTIFICATION:
+        elif msg['type'] == RPCMessageType.WARNING:
             message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg)
 
-        elif msg['type'] == RPCMessageType.STARTUP_NOTIFICATION:
+        elif msg['type'] == RPCMessageType.STARTUP:
             message = '{status}'.format(**msg)
 
         else:
@@ -294,6 +307,7 @@ class Telegram(RPCHandler):
 
             messages = []
             for r in results:
+                r['open_date_hum'] = arrow.get(r['open_date']).humanize()
                 lines = [
                     "*Trade ID:* `{trade_id}` `(since {open_date_hum})`",
                     "*Current Pair:* {pair}",
@@ -695,14 +709,18 @@ class Telegram(RPCHandler):
         """
         try:
             trades = self._rpc._rpc_performance()
-            stats = '\n'.join('{index}.\t<code>{pair}\t{profit:.2f}% ({count})</code>'.format(
-                index=i + 1,
-                pair=trade['pair'],
-                profit=trade['profit'],
-                count=trade['count']
-            ) for i, trade in enumerate(trades))
-            message = '<b>Performance:</b>\n{}'.format(stats)
-            self._send_msg(message, parse_mode=ParseMode.HTML)
+            output = "<b>Performance:</b>\n"
+            for i, trade in enumerate(trades):
+                stat_line = (f"{i+1}.\t <code>{trade['pair']}\t{trade['profit']:.2f}% "
+                             f"({trade['count']})</code>\n")
+
+                if len(output + stat_line) >= MAX_TELEGRAM_MESSAGE_LENGTH:
+                    self._send_msg(output, parse_mode=ParseMode.HTML)
+                    output = stat_line
+                else:
+                    output += stat_line
+
+            self._send_msg(output, parse_mode=ParseMode.HTML)
         except RPCException as e:
             self._send_msg(str(e))
 

freqtrade/rpc/webhook.py

@@ -45,17 +45,21 @@ class Webhook(RPCHandler):
         """ Send a message to telegram channel """
         try:
 
-            if msg['type'] == RPCMessageType.BUY_NOTIFICATION:
+            if msg['type'] == RPCMessageType.BUY:
                 valuedict = self._config['webhook'].get('webhookbuy', None)
-            elif msg['type'] == RPCMessageType.BUY_CANCEL_NOTIFICATION:
+            elif msg['type'] == RPCMessageType.BUY_CANCEL:
                 valuedict = self._config['webhook'].get('webhookbuycancel', None)
-            elif msg['type'] == RPCMessageType.SELL_NOTIFICATION:
+            elif msg['type'] == RPCMessageType.BUY_FILL:
+                valuedict = self._config['webhook'].get('webhookbuyfill', None)
+            elif msg['type'] == RPCMessageType.SELL:
                 valuedict = self._config['webhook'].get('webhooksell', None)
-            elif msg['type'] == RPCMessageType.SELL_CANCEL_NOTIFICATION:
+            elif msg['type'] == RPCMessageType.SELL_FILL:
+                valuedict = self._config['webhook'].get('webhooksellfill', None)
+            elif msg['type'] == RPCMessageType.SELL_CANCEL:
                 valuedict = self._config['webhook'].get('webhooksellcancel', None)
-            elif msg['type'] in (RPCMessageType.STATUS_NOTIFICATION,
-                                 RPCMessageType.STARTUP_NOTIFICATION,
-                                 RPCMessageType.WARNING_NOTIFICATION):
+            elif msg['type'] in (RPCMessageType.STATUS,
+                                 RPCMessageType.STARTUP,
+                                 RPCMessageType.WARNING):
                 valuedict = self._config['webhook'].get('webhookstatus', None)
             else:
                 raise NotImplementedError('Unknown message type: {}'.format(msg['type']))

freqtrade/strategy/__init__.py

@@ -1,5 +1,7 @@
 # flake8: noqa: F401
 from freqtrade.exchange import (timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date,
                                 timeframe_to_prev_date, timeframe_to_seconds)
+from freqtrade.strategy.hyper import (CategoricalParameter, DecimalParameter, IntParameter,
+                                      RealParameter)
 from freqtrade.strategy.interface import IStrategy
 from freqtrade.strategy.strategy_helper import merge_informative_pair, stoploss_from_open

freqtrade/strategy/hyper.py

@@ -0,0 +1,297 @@
+"""
+IHyperStrategy interface, hyperoptable Parameter class.
+This module defines a base class for auto-hyperoptable strategies.
+"""
+import logging
+from abc import ABC, abstractmethod
+from contextlib import suppress
+from typing import Any, Dict, Iterator, Optional, Sequence, Tuple, Union
+
+
+with suppress(ImportError):
+    from skopt.space import Integer, Real, Categorical
+    from freqtrade.optimize.space import SKDecimal
+
+from freqtrade.exceptions import OperationalException
+from freqtrade.state import RunMode
+
+
+logger = logging.getLogger(__name__)
+
+
+class BaseParameter(ABC):
+    """
+    Defines a parameter that can be optimized by hyperopt.
+    """
+    category: Optional[str]
+    default: Any
+    value: Any
+    hyperopt: bool = False
+
+    def __init__(self, *, default: Any, space: Optional[str] = None,
+                 optimize: bool = True, load: bool = True, **kwargs):
+        """
+        Initialize hyperopt-optimizable parameter.
+        :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if
+         parameter field
+         name is prefixed with 'buy_' or 'sell_'.
+        :param optimize: Include parameter in hyperopt optimizations.
+        :param load: Load parameter value from {space}_params.
+        :param kwargs: Extra parameters to skopt.space.(Integer|Real|Categorical).
+        """
+        if 'name' in kwargs:
+            raise OperationalException(
+                'Name is determined by parameter field name and can not be specified manually.')
+        self.category = space
+        self._space_params = kwargs
+        self.value = default
+        self.optimize = optimize
+        self.load = load
+
+    def __repr__(self):
+        return f'{self.__class__.__name__}({self.value})'
+
+    @abstractmethod
+    def get_space(self, name: str) -> Union['Integer', 'Real', 'SKDecimal', 'Categorical']:
+        """
+        Get-space - will be used by Hyperopt to get the hyperopt Space
+        """
+
+
+class NumericParameter(BaseParameter):
+    """ Internal parameter used for Numeric purposes """
+    float_or_int = Union[int, float]
+    default: float_or_int
+    value: float_or_int
+
+    def __init__(self, low: Union[float_or_int, Sequence[float_or_int]],
+                 high: Optional[float_or_int] = None, *, default: float_or_int,
+                 space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs):
+        """
+        Initialize hyperopt-optimizable numeric parameter.
+        Cannot be instantiated, but provides the validation for other numeric parameters
+        :param low: Lower end (inclusive) of optimization space or [low, high].
+        :param high: Upper end (inclusive) of optimization space.
+                     Must be none of entire range is passed first parameter.
+        :param default: A default value.
+        :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if
+                      parameter fieldname is prefixed with 'buy_' or 'sell_'.
+        :param optimize: Include parameter in hyperopt optimizations.
+        :param load: Load parameter value from {space}_params.
+        :param kwargs: Extra parameters to skopt.space.*.
+        """
+        if high is not None and isinstance(low, Sequence):
+            raise OperationalException(f'{self.__class__.__name__} space invalid.')
+        if high is None or isinstance(low, Sequence):
+            if not isinstance(low, Sequence) or len(low) != 2:
+                raise OperationalException(f'{self.__class__.__name__} space must be [low, high]')
+            self.low, self.high = low
+        else:
+            self.low = low
+            self.high = high
+
+        super().__init__(default=default, space=space, optimize=optimize,
+                         load=load, **kwargs)
+
+
+class IntParameter(NumericParameter):
+    default: int
+    value: int
+
+    def __init__(self, low: Union[int, Sequence[int]], high: Optional[int] = None, *, default: int,
+                 space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs):
+        """
+        Initialize hyperopt-optimizable integer parameter.
+        :param low: Lower end (inclusive) of optimization space or [low, high].
+        :param high: Upper end (inclusive) of optimization space.
+                     Must be none of entire range is passed first parameter.
+        :param default: A default value.
+        :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if
+                      parameter fieldname is prefixed with 'buy_' or 'sell_'.
+        :param optimize: Include parameter in hyperopt optimizations.
+        :param load: Load parameter value from {space}_params.
+        :param kwargs: Extra parameters to skopt.space.Integer.
+        """
+
+        super().__init__(low=low, high=high, default=default, space=space, optimize=optimize,
+                         load=load, **kwargs)
+
+    def get_space(self, name: str) -> 'Integer':
+        """
+        Create skopt optimization space.
+        :param name: A name of parameter field.
+        """
+        return Integer(low=self.low, high=self.high, name=name, **self._space_params)
+
+    @property
+    def range(self):
+        """
+        Get each value in this space as list.
+        Returns a List from low to high (inclusive) in Hyperopt mode.
+        Returns a List with 1 item (`value`) in "non-hyperopt" mode, to avoid
+        calculating 100ds of indicators.
+        """
+        if self.hyperopt:
+            # Scikit-optimize ranges are "inclusive", while python's "range" is exclusive
+            return range(self.low, self.high + 1)
+        else:
+            return range(self.value, self.value + 1)
+
+
+class RealParameter(NumericParameter):
+    default: float
+    value: float
+
+    def __init__(self, low: Union[float, Sequence[float]], high: Optional[float] = None, *,
+                 default: float, space: Optional[str] = None, optimize: bool = True,
+                 load: bool = True, **kwargs):
+        """
+        Initialize hyperopt-optimizable floating point parameter with unlimited precision.
+        :param low: Lower end (inclusive) of optimization space or [low, high].
+        :param high: Upper end (inclusive) of optimization space.
+                     Must be none if entire range is passed first parameter.
+        :param default: A default value.
+        :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if
+                      parameter fieldname is prefixed with 'buy_' or 'sell_'.
+        :param optimize: Include parameter in hyperopt optimizations.
+        :param load: Load parameter value from {space}_params.
+        :param kwargs: Extra parameters to skopt.space.Real.
+        """
+        super().__init__(low=low, high=high, default=default, space=space, optimize=optimize,
+                         load=load, **kwargs)
+
+    def get_space(self, name: str) -> 'Real':
+        """
+        Create skopt optimization space.
+        :param name: A name of parameter field.
+        """
+        return Real(low=self.low, high=self.high, name=name, **self._space_params)
+
+
+class DecimalParameter(NumericParameter):
+    default: float
+    value: float
+
+    def __init__(self, low: Union[float, Sequence[float]], high: Optional[float] = None, *,
+                 default: float, decimals: int = 3, space: Optional[str] = None,
+                 optimize: bool = True, load: bool = True, **kwargs):
+        """
+        Initialize hyperopt-optimizable decimal parameter with a limited precision.
+        :param low: Lower end (inclusive) of optimization space or [low, high].
+        :param high: Upper end (inclusive) of optimization space.
+                     Must be none if entire range is passed first parameter.
+        :param default: A default value.
+        :param decimals: A number of decimals after floating point to be included in testing.
+        :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if
+                      parameter fieldname is prefixed with 'buy_' or 'sell_'.
+        :param optimize: Include parameter in hyperopt optimizations.
+        :param load: Load parameter value from {space}_params.
+        :param kwargs: Extra parameters to skopt.space.Integer.
+        """
+        self._decimals = decimals
+        default = round(default, self._decimals)
+
+        super().__init__(low=low, high=high, default=default, space=space, optimize=optimize,
+                         load=load, **kwargs)
+
+    def get_space(self, name: str) -> 'SKDecimal':
+        """
+        Create skopt optimization space.
+        :param name: A name of parameter field.
+        """
+        return SKDecimal(low=self.low, high=self.high, decimals=self._decimals, name=name,
+                         **self._space_params)
+
+
+class CategoricalParameter(BaseParameter):
+    default: Any
+    value: Any
+    opt_range: Sequence[Any]
+
+    def __init__(self, categories: Sequence[Any], *, default: Optional[Any] = None,
+                 space: Optional[str] = None, optimize: bool = True, load: bool = True, **kwargs):
+        """
+        Initialize hyperopt-optimizable parameter.
+        :param categories: Optimization space, [a, b, ...].
+        :param default: A default value. If not specified, first item from specified space will be
+         used.
+        :param space: A parameter category. Can be 'buy' or 'sell'. This parameter is optional if
+         parameter field
+         name is prefixed with 'buy_' or 'sell_'.
+        :param optimize: Include parameter in hyperopt optimizations.
+        :param load: Load parameter value from {space}_params.
+        :param kwargs: Extra parameters to skopt.space.Categorical.
+        """
+        if len(categories) < 2:
+            raise OperationalException(
+                'CategoricalParameter space must be [a, b, ...] (at least two parameters)')
+        self.opt_range = categories
+        super().__init__(default=default, space=space, optimize=optimize,
+                         load=load, **kwargs)
+
+    def get_space(self, name: str) -> 'Categorical':
+        """
+        Create skopt optimization space.
+        :param name: A name of parameter field.
+        """
+        return Categorical(self.opt_range, name=name, **self._space_params)
+
+
+class HyperStrategyMixin(object):
+    """
+    A helper base class which allows HyperOptAuto class to reuse implementations of of buy/sell
+     strategy logic.
+    """
+
+    def __init__(self, config: Dict[str, Any], *args, **kwargs):
+        """
+        Initialize hyperoptable strategy mixin.
+        """
+        self._load_hyper_params(config.get('runmode') == RunMode.HYPEROPT)
+
+    def enumerate_parameters(self, category: str = None) -> Iterator[Tuple[str, BaseParameter]]:
+        """
+        Find all optimizeable parameters and return (name, attr) iterator.
+        :param category:
+        :return:
+        """
+        if category not in ('buy', 'sell', None):
+            raise OperationalException('Category must be one of: "buy", "sell", None.')
+        for attr_name in dir(self):
+            if not attr_name.startswith('__'):  # Ignore internals, not strictly necessary.
+                attr = getattr(self, attr_name)
+                if issubclass(attr.__class__, BaseParameter):
+                    if (category and attr_name.startswith(category + '_')
+                            and attr.category is not None and attr.category != category):
+                        raise OperationalException(
+                            f'Inconclusive parameter name {attr_name}, category: {attr.category}.')
+                    if (category is None or category == attr.category or
+                            (attr_name.startswith(category + '_') and attr.category is None)):
+                        yield attr_name, attr
+
+    def _load_hyper_params(self, hyperopt: bool = False) -> None:
+        """
+        Load Hyperoptable parameters
+        """
+        self._load_params(getattr(self, 'buy_params', None), 'buy', hyperopt)
+        self._load_params(getattr(self, 'sell_params', None), 'sell', hyperopt)
+
+    def _load_params(self, params: dict, space: str, hyperopt: bool = False) -> None:
+        """
+        Set optimizeable parameter values.
+        :param params: Dictionary with new parameter values.
+        """
+        if not params:
+            logger.info(f"No params for {space} found, using default values.")
+
+        for attr_name, attr in self.enumerate_parameters():
+            attr.hyperopt = hyperopt
+            if params and attr_name in params:
+                if attr.load:
+                    attr.value = params[attr_name]
+                    logger.info(f'Strategy Parameter: {attr_name} = {attr.value}')
+                else:
+                    logger.warning(f'Parameter "{attr_name}" exists, but is disabled. '
+                                   f'Default value "{attr.value}" used.')
+            else:
+                logger.info(f'Strategy Parameter(default): {attr_name} = {attr.value}')

freqtrade/strategy/interface.py

@@ -18,6 +18,7 @@ from freqtrade.exceptions import OperationalException, StrategyError
 from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds
 from freqtrade.exchange.exchange import timeframe_to_next_date
 from freqtrade.persistence import PairLocks, Trade
+from freqtrade.strategy.hyper import HyperStrategyMixin
 from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper
 from freqtrade.wallets import Wallets
 
@@ -59,7 +60,7 @@ class SellCheckTuple(NamedTuple):
     sell_type: SellType
 
 
-class IStrategy(ABC):
+class IStrategy(ABC, HyperStrategyMixin):
     """
     Interface for freqtrade strategies
     Defines the mandatory structure must follow any custom strategies
@@ -140,6 +141,7 @@ class IStrategy(ABC):
         self.config = config
         # Dict to determine if analysis is necessary
         self._last_candle_seen_per_pair: Dict[str, datetime] = {}
+        super().__init__(config)
 
     @abstractmethod
     def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:

freqtrade/templates/base_config.json.j2

@@ -54,15 +54,15 @@
         "chat_id": "{{ telegram_chat_id }}"
     },
     "api_server": {
-        "enabled": false,
-        "listen_ip_address": "127.0.0.1",
+        "enabled": {{ api_server | lower }},
+        "listen_ip_address": "{{ api_server_listen_addr | default("127.0.0.1", true) }}",
         "listen_port": 8080,
         "verbosity": "error",
         "enable_openapi": false,
-        "jwt_secret_key": "somethingrandom",
+        "jwt_secret_key": "{{ api_server_jwt_key }}",
         "CORS_origins": [],
-        "username": "",
-        "password": ""
+        "username": "{{ api_server_username }}",
+        "password": "{{ api_server_password }}"
     },
     "bot_name": "freqtrade",
     "initial_state": "running",

freqtrade/templates/base_strategy.py.j2

@@ -1,4 +1,5 @@
 # pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
+# flake8: noqa: F401
 
 # --- Do not remove these libs ---
 import numpy as np  # noqa
@@ -6,6 +7,7 @@ import pandas as pd  # noqa
 from pandas import DataFrame
 
 from freqtrade.strategy import IStrategy
+from freqtrade.strategy import CategoricalParameter, DecimalParameter, IntParameter
 
 # --------------------------------
 # Add your lib to import here
@@ -16,7 +18,7 @@ import freqtrade.vendor.qtpylib.indicators as qtpylib
 class {{ strategy }}(IStrategy):
     """
     This is a strategy template to get you started.
-    More information in https://github.com/freqtrade/freqtrade/blob/develop/docs/bot-optimization.md
+    More information in https://www.freqtrade.io/en/latest/strategy-customization/
 
     You can:
         :return: a Dataframe with all mandatory indicators for the strategies
@@ -26,8 +28,9 @@ class {{ strategy }}(IStrategy):
 
     You must keep:
     - the lib in the section "Do not remove these libs"
-    - the prototype for the methods: minimal_roi, stoploss, populate_indicators, populate_buy_trend,
-    populate_sell_trend, hyperopt_space, buy_strategy_generator
+    - the methods: populate_indicators, populate_buy_trend, populate_sell_trend
+    You should keep:
+    - timeframe, minimal_roi, stoploss, trailing_*
     """
     # Strategy interface version - allow new iterations of the strategy interface.
     # Check the documentation or the Sample strategy to get the latest version.

freqtrade/templates/sample_hyperopt_advanced.py

@@ -7,7 +7,7 @@ from typing import Any, Callable, Dict, List
 import numpy as np  # noqa
 import pandas as pd  # noqa
 from pandas import DataFrame
-from skopt.space import Categorical, Dimension, Integer, Real  # noqa
+from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real  # noqa
 
 from freqtrade.optimize.hyperopt_interface import IHyperOpt
 
@@ -223,9 +223,9 @@ class AdvancedSampleHyperOpt(IHyperOpt):
             Integer(10, 120, name='roi_t1'),
             Integer(10, 60, name='roi_t2'),
             Integer(10, 40, name='roi_t3'),
-            Real(0.01, 0.04, name='roi_p1'),
-            Real(0.01, 0.07, name='roi_p2'),
-            Real(0.01, 0.20, name='roi_p3'),
+            SKDecimal(0.01, 0.04, decimals=3, name='roi_p1'),
+            SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'),
+            SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'),
         ]
 
     @staticmethod
@@ -237,7 +237,7 @@ class AdvancedSampleHyperOpt(IHyperOpt):
         'stoploss' optimization hyperspace.
         """
         return [
-            Real(-0.35, -0.02, name='stoploss'),
+            SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'),
         ]
 
     @staticmethod
@@ -256,14 +256,14 @@ class AdvancedSampleHyperOpt(IHyperOpt):
             # other 'trailing' hyperspace parameters.
             Categorical([True], name='trailing_stop'),
 
-            Real(0.01, 0.35, name='trailing_stop_positive'),
+            SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'),
 
             # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive',
             # so this intermediate parameter is used as the value of the difference between
             # them. The value of the 'trailing_stop_positive_offset' is constructed in the
             # generate_trailing_params() method.
             # This is similar to the hyperspace dimensions used for constructing the ROI tables.
-            Real(0.001, 0.1, name='trailing_stop_positive_offset_p1'),
+            SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'),
 
             Categorical([True, False], name='trailing_only_offset_is_reached'),
         ]