Merge branch 'develop' into clear-caplog-freqtradebot

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,14 +2,16 @@ Thank you for sending your pull request. But first, have you included
 unit tests, and is your code PEP8 conformant? [More details](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md)
 
 ## Summary
+
 Explain in one sentence the goal of this PR
 
 Solve the issue: #___
 
 ## Quick changelog
 
-- <change log #1>
+- <change log 1>
-- <change log #2>
+- <change log 1>
 
 ## What's new?
+
 *Explain in details what this PR solve or improve. You can include visuals.* 

Dockerfile

@@ -13,7 +13,7 @@ RUN mkdir /freqtrade \
   && apt-get update \
   && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev  \
   && apt-get clean \
-  && useradd -u 1000 -G sudo -U -m ftuser \
+  && useradd -u 1000 -G sudo -U -m -s /bin/bash ftuser \
   && chown ftuser:ftuser /freqtrade \
   # Allow sudoers
   && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers

build_helpers/install_ta-lib.sh

@@ -12,9 +12,12 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then
   && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \
   && ./configure --prefix=${INSTALL_LOC}/ \
   && make -j$(nproc) \
-  && which sudo && sudo make install || make install \
+  && which sudo && sudo make install || make install
-  && cd ..
+  if [ -x "$(command -v apt-get)" ]; then
+    echo "Updating library path using ldconfig"
+    sudo ldconfig
+  fi
+  cd .. && rm -rf ./ta-lib/
 else
   echo "TA-lib already installed, skipping installation"
 fi
-#  && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \

docs/advanced-hyperopt.md

@@ -80,7 +80,7 @@ To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_sp
 class MyAwesomeStrategy(IStrategy):
     class HyperOpt:
         # Define a custom stoploss space.
-        def stoploss_space(self):
+        def stoploss_space():
             return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')]
 ```
 

docs/configuration.md

@@ -444,8 +444,8 @@ The possible values are: `gtc` (default), `fok` or `ioc`.
 ```
 
 !!! Warning
-    This is ongoing work. For now, it is supported only for binance.
+    This is ongoing work. For now, it is supported only for binance and kucoin.
-    Please don't change the default value unless you know what you are doing and have researched the impact of using different values.
+    Please don't change the default value unless you know what you are doing and have researched the impact of using different values for your particular exchange.
 
 ### Exchange configuration
 

docs/edge.md

@@ -3,7 +3,7 @@
 The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss.
 
 !!! Warning
-    WHen using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data.
+    When using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data.
 
 !!! Note
     `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file.

docs/exchanges.md

@@ -4,6 +4,8 @@ This page combines common gotchas and informations which are exchange-specific a
 
 ## Binance
 
+Binance supports [time_in_force](configuration.md#understand-order_time_in_force).
+
 !!! Tip "Stoploss on Exchange"
     Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it.
 
@@ -113,8 +115,12 @@ Kucoin requires a passphrase for each api key, you will therefore need to add th
     "key": "your_exchange_key",
     "secret": "your_exchange_secret",
     "password": "your_exchange_api_key_password",
+    // ...
+}
 ```
 
+Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force).
+
 ### Kucoin Blacklists
 
 For Kucoin, please add `"KCS/<STAKE>"` to your blacklist to avoid issues.
@@ -158,6 +164,8 @@ For example, to test the order type `FOK` with Kraken, and modify candle limit t
         "order_time_in_force": ["gtc", "fok"],
         "ohlcv_candle_limit": 200
         }
+    //...
+}
 ```
 
 !!! Warning

docs/requirements-docs.txt

@@ -1,4 +1,4 @@
 mkdocs==1.2.2
-mkdocs-material==7.2.5
+mkdocs-material==7.2.6
 mdx_truly_sane_lists==1.2
 pymdown-extensions==8.2

freqtrade/__init__.py

@@ -22,7 +22,7 @@ if __version__ == 'develop':
         #  subprocess.check_output(
         #     ['git', 'log', '--format="%h"', '-n 1'],
         #     stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"')
-    except Exception:
+    except Exception:  # pragma: no cover
         # git not available, ignore
         try:
             # Try Fallback to freqtrade_commit file (created by CI while building docker image)

freqtrade/commands/build_config_commands.py

@@ -61,13 +61,13 @@ def ask_user_config() -> Dict[str, Any]:
             "type": "text",
             "name": "stake_currency",
             "message": "Please insert your stake currency:",
-            "default": 'BTC',
+            "default": 'USDT',
         },
         {
             "type": "text",
             "name": "stake_amount",
             "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):",
-            "default": "0.01",
+            "default": "100",
             "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val),
             "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
             if val == UNLIMITED_STAKE_AMOUNT
@@ -105,6 +105,8 @@ def ask_user_config() -> Dict[str, Any]:
                 "bittrex",
                 "kraken",
                 "ftx",
+                "kucoin",
+                "gateio",
                 Separator(),
                 "other",
             ],
@@ -128,6 +130,12 @@ def ask_user_config() -> Dict[str, Any]:
             "message": "Insert Exchange Secret",
             "when": lambda x: not x['dry_run']
         },
+        {
+            "type": "password",
+            "name": "exchange_key_password",
+            "message": "Insert Exchange API Key password",
+            "when": lambda x: not x['dry_run'] and x['exchange_name'] == 'kucoin'
+        },
         {
             "type": "confirm",
             "name": "telegram",

freqtrade/exchange/binance.py

@@ -18,6 +18,7 @@ class Binance(Exchange):
     _ft_has: Dict = {
         "stoploss_on_exchange": True,
         "order_time_in_force": ['gtc', 'fok', 'ioc'],
+        "time_in_force_parameter": "timeInForce",
         "ohlcv_candle_limit": 1000,
         "trades_pagination": "id",
         "trades_pagination_arg": "fromId",

freqtrade/exchange/exchange.py

@@ -54,12 +54,16 @@ class Exchange:
     # Parameters to add directly to buy/sell calls (like agreeing to trading agreement)
     _params: Dict = {}
 
+    # Additional headers - added to the ccxt object
+    _headers: Dict = {}
+
     # Dict to specify which options each exchange implements
     # This defines defaults, which can be selectively overridden by subclasses using _ft_has
     # or by specifying them in the configuration.
     _ft_has_default: Dict = {
         "stoploss_on_exchange": False,
         "order_time_in_force": ["gtc"],
+        "time_in_force_parameter": "timeInForce",
         "ohlcv_params": {},
         "ohlcv_candle_limit": 500,
         "ohlcv_partial_candle": True,
@@ -169,7 +173,7 @@ class Exchange:
             asyncio.get_event_loop().run_until_complete(self._api_async.close())
 
     def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt,
-                   ccxt_kwargs: dict = None) -> ccxt.Exchange:
+                   ccxt_kwargs: Dict = {}) -> ccxt.Exchange:
         """
         Initialize ccxt with given config and return valid
         ccxt instance.
@@ -188,6 +192,10 @@ class Exchange:
         }
         if ccxt_kwargs:
             logger.info('Applying additional ccxt config: %s', ccxt_kwargs)
+        if self._headers:
+            # Inject static headers after the above output to not confuse users.
+            ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs)
+        if ccxt_kwargs:
             ex_config.update(ccxt_kwargs)
         try:
 
@@ -716,7 +724,8 @@ class Exchange:
 
         params = self._params.copy()
         if time_in_force != 'gtc' and ordertype != 'market':
-            params.update({'timeInForce': time_in_force})
+            param = self._ft_has.get('time_in_force_parameter', '')
+            params.update({param: time_in_force})
 
         try:
             # Set the precision for amount and price(rate) as accepted by the exchange

freqtrade/exchange/kucoin.py

@@ -21,4 +21,6 @@ class Kucoin(Exchange):
     _ft_has: Dict = {
         "l2_limit_range": [20, 100],
         "l2_limit_range_required": False,
+        "order_time_in_force": ['gtc', 'fok', 'ioc'],
+        "time_in_force_parameter": "timeInForce",
     }

freqtrade/loggers.py

@@ -87,7 +87,7 @@ def setup_logging(config: Dict[str, Any]) -> None:
             # syslog config. The messages should be equal for this.
             handler_sl.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
             logging.root.addHandler(handler_sl)
-        elif s[0] == 'journald':
+        elif s[0] == 'journald':  # pragma: no cover
             try:
                 from systemd.journal import JournaldLogHandler
             except ImportError:

freqtrade/main.py

@@ -9,7 +9,7 @@ from typing import Any, List
 
 
 # check min. python version
-if sys.version_info < (3, 7):
+if sys.version_info < (3, 7):  # pragma: no cover
     sys.exit("Freqtrade requires Python version >= 3.7")
 
 from freqtrade.commands import Arguments
@@ -46,7 +46,7 @@ def main(sysargv: List[str] = None) -> None:
                 "`freqtrade --help` or `freqtrade <command> --help`."
             )
 
-    except SystemExit as e:
+    except SystemExit as e:  # pragma: no cover
         return_code = e
     except KeyboardInterrupt:
         logger.info('SIGINT received, aborting ...')
@@ -60,5 +60,5 @@ def main(sysargv: List[str] = None) -> None:
         sys.exit(return_code)
 
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     main()

freqtrade/plugins/pairlist/pairlist_helpers.py

@@ -17,7 +17,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
     if keep_invalid:
         for pair_wc in wildcardpl:
             try:
-                comp = re.compile(pair_wc)
+                comp = re.compile(pair_wc, re.IGNORECASE)
                 result_partial = [
                     pair for pair in available_pairs if re.fullmatch(comp, pair)
                 ]
@@ -33,7 +33,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
     else:
         for pair_wc in wildcardpl:
             try:
-                comp = re.compile(pair_wc)
+                comp = re.compile(pair_wc, re.IGNORECASE)
                 result += [
                     pair for pair in available_pairs if re.fullmatch(comp, pair)
                 ]

freqtrade/rpc/api_server/uvicorn_threaded.py

@@ -5,6 +5,20 @@ import time
 import uvicorn
 
 
+def asyncio_setup() -> None:  # pragma: no cover
+    # Set eventloop for win32 setups
+    # Reverts a change done in uvicorn 0.15.0 - which now sets the eventloop
+    # via policy.
+    import sys
+
+    if sys.version_info >= (3, 8) and sys.platform == "win32":
+        import asyncio
+        import selectors
+        selector = selectors.SelectSelector()
+        loop = asyncio.SelectorEventLoop(selector)
+        asyncio.set_event_loop(loop)
+
+
 class UvicornServer(uvicorn.Server):
     """
     Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742
@@ -28,7 +42,7 @@ class UvicornServer(uvicorn.Server):
         try:
             import uvloop  # noqa
         except ImportError:  # pragma: no cover
-            from uvicorn.loops.asyncio import asyncio_setup
+
             asyncio_setup()
         else:
             asyncio.set_event_loop(uvloop.new_event_loop())

freqtrade/rpc/rpc.py

@@ -403,6 +403,9 @@ class RPC:
         # Doing the sum is not right - overall profit needs to be based on initial capital
         profit_all_ratio_sum = sum(profit_all_ratio) if profit_all_ratio else 0.0
         starting_balance = self._freqtrade.wallets.get_starting_balance()
+        profit_closed_ratio_fromstart = 0
+        profit_all_ratio_fromstart = 0
+        if starting_balance:
             profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance
             profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance
 

freqtrade/templates/base_config.json.j2

@@ -1,3 +1,10 @@
+{%set volume_pairlist = '{
+            "method": "VolumePairList",
+            "number_assets": 20,
+            "sort_key": "quoteVolume",
+            "min_value": 0,
+            "refresh_period": 1800
+        }' %}
 {
     "max_open_trades": {{ max_open_trades }},
     "stake_currency": "{{ stake_currency }}",
@@ -29,7 +36,7 @@
     },
     {{ exchange | indent(4) }},
     "pairlists": [
-        {"method": "StaticPairList"}
+        {{ '{"method": "StaticPairList"}' if exchange_name == 'bittrex' else  volume_pairlist }}
     ],
     "edge": {
         "enabled": false,

freqtrade/templates/subtemplates/exchange_binance.j2

@@ -8,34 +8,8 @@
         "rateLimit": 200
     },
     "pair_whitelist": [
-        "ALGO/BTC",
-        "ATOM/BTC",
-        "BAT/BTC",
-        "BCH/BTC",
-        "BRD/BTC",
-        "EOS/BTC",
-        "ETH/BTC",
-        "IOTA/BTC",
-        "LINK/BTC",
-        "LTC/BTC",
-        "NEO/BTC",
-        "NXS/BTC",
-        "XMR/BTC",
-        "XRP/BTC",
-        "XTZ/BTC"
     ],
     "pair_blacklist": [
-        "BNB/BTC",
+        "BNB/.*"
-        "BNB/BUSD",
-        "BNB/ETH",
-        "BNB/EUR",
-        "BNB/NGN",
-        "BNB/PAX",
-        "BNB/RUB",
-        "BNB/TRY",
-        "BNB/TUSD",
-        "BNB/USDC",
-        "BNB/USDS",
-        "BNB/USDT"
     ]
 }

freqtrade/templates/subtemplates/exchange_bittrex.j2

@@ -15,16 +15,6 @@
         "rateLimit": 500
     },
     "pair_whitelist": [
-        "ETH/BTC",
-        "LTC/BTC",
-        "ETC/BTC",
-        "DASH/BTC",
-        "ZEC/BTC",
-        "XLM/BTC",
-        "XRP/BTC",
-        "TRX/BTC",
-        "ADA/BTC",
-        "XMR/BTC"
     ],
     "pair_blacklist": [
     ]

freqtrade/templates/subtemplates/exchange_kraken.j2

@@ -7,28 +7,10 @@
     "ccxt_async_config": {
         "enableRateLimit": true,
         "rateLimit": 1000
+        // Enable the below for downoading data.
+        //"rateLimit": 3100
     },
     "pair_whitelist": [
-        "ADA/EUR",
-        "ATOM/EUR",
-        "BAT/EUR",
-        "BCH/EUR",
-        "BTC/EUR",
-        "DAI/EUR",
-        "DASH/EUR",
-        "EOS/EUR",
-        "ETC/EUR",
-        "ETH/EUR",
-        "LINK/EUR",
-        "LTC/EUR",
-        "QTUM/EUR",
-        "REP/EUR",
-        "WAVES/EUR",
-        "XLM/EUR",
-        "XMR/EUR",
-        "XRP/EUR",
-        "XTZ/EUR",
-        "ZEC/EUR"
     ],
     "pair_blacklist": [
 

freqtrade/templates/subtemplates/exchange_kucoin.j2

@@ -0,0 +1,18 @@
+"exchange": {
+    "name": "{{ exchange_name | lower }}",
+    "key": "{{ exchange_key }}",
+    "secret": "{{ exchange_secret }}",
+    "password": "{{ exchange_key_password }}",
+    "ccxt_config": {
+        "enableRateLimit": true
+        "rateLimit": 200
+    },
+    "ccxt_async_config": {
+        "enableRateLimit": true,
+        "rateLimit": 200
+    },
+    "pair_whitelist": [
+    ],
+    "pair_blacklist": [
+    ]
+}

mkdocs.yml

@@ -23,10 +23,10 @@ nav:
     - Hyperopt: hyperopt.md
     - Utility Sub-commands: utils.md
     - Plotting: plotting.md
+    - Exchange-specific Notes: exchanges.md
     - Data Analysis:
         - Jupyter Notebooks: data-analysis.md
         - Strategy analysis: strategy_analysis_example.md
-    - Exchange-specific Notes: exchanges.md
     - Advanced Topics:
         - Advanced Post-installation Tasks: advanced-setup.md
         - Edge Positioning: edge.md

requirements-dev.txt

@@ -8,7 +8,7 @@ flake8==3.9.2
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==4.4.1
 mypy==0.910
-pytest==6.2.4
+pytest==6.2.5
 pytest-asyncio==0.15.1
 pytest-cov==2.12.1
 pytest-mock==3.6.1

requirements-plot.txt

@@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt
 
-plotly==5.3.0
+plotly==5.3.1
 

requirements.txt

@@ -1,7 +1,7 @@
 numpy==1.21.2
 pandas==1.3.2
 
-ccxt==1.55.56
+ccxt==1.55.83
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==3.4.8
 aiohttp==3.7.4.post0

setup.sh

@@ -95,19 +95,7 @@ function install_talib() {
         return
     fi
 
-    cd build_helpers
+    cd build_helpers && ./install_ta-lib.sh && cd ..
-    tar zxvf ta-lib-0.4.0-src.tar.gz
-    cd ta-lib
-    sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h
-    ./configure --prefix=/usr/local
-    make
-    sudo make install
-    if [ -x "$(command -v apt-get)" ]; then
-        echo "Updating library path using ldconfig"
-        sudo ldconfig
-    fi
-    cd .. && rm -rf ./ta-lib/
-    cd ..
 }
 
 function install_mac_newer_python_dependencies() {    

tests/exchange/test_exchange.py

@@ -108,6 +108,13 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog):
     assert hasattr(ex._api_async, 'TestKWARG')
     assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog)
     assert log_has(asynclogmsg, caplog)
+    # Test additional headers case
+    Exchange._headers = {'hello': 'world'}
+    ex = Exchange(conf)
+
+    assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog)
+    assert ex._api.headers == {'hello': 'world'}
+    Exchange._headers = {}
 
 
 def test_destroy(default_conf, mocker, caplog):

tests/strategy/test_interface.py

@@ -735,11 +735,16 @@ def test_auto_hyperopt_interface(default_conf):
     PairLocks.timeframe = default_conf['timeframe']
     strategy = StrategyResolver.load_strategy(default_conf)
 
+    with pytest.raises(OperationalException):
+        next(strategy.enumerate_parameters('deadBeef'))
+
     assert strategy.buy_rsi.value == strategy.buy_params['buy_rsi']
     # PlusDI is NOT in the buy-params, so default should be used
     assert strategy.buy_plusdi.value == 0.5
     assert strategy.sell_rsi.value == strategy.sell_params['sell_rsi']
 
+    assert repr(strategy.sell_rsi) == 'IntParameter(74)'
+
     # Parameter is disabled - so value from sell_param dict will NOT be used.
     assert strategy.sell_minusdi.value == 0.5
     all_params = strategy.detect_all_parameters()