Merge branch 'develop' into clear-caplog-freqtradebot

2021-09-07 21:55:00 -06:00 · 2021-09-07 21:55:00 -06:00 · 93fcaac19f
commit 93fcaac19f
parent 79ca6135a2 880474594e
30 changed files with 121 additions and 100 deletions
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@ -2,14 +2,16 @@ Thank you for sending your pull request. But first, have you included
 unit tests, and is your code PEP8 conformant? [More details](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md)

 ## Summary
+
 Explain in one sentence the goal of this PR

 Solve the issue: #___

 ## Quick changelog

- <change log #1>
- <change log #2>
+- <change log 1>
+- <change log 1>

 ## What's new?
+
 *Explain in details what this PR solve or improve. You can include visuals.* 
--- a/2
+++ b/2
@ -13,7 +13,7 @@ RUN mkdir /freqtrade \
  && apt-get update \
  && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev  \
  && apt-get clean \
-  && useradd -u 1000 -G sudo -U -m ftuser \
+  && useradd -u 1000 -G sudo -U -m -s /bin/bash ftuser \
  && chown ftuser:ftuser /freqtrade \
  # Allow sudoers
  && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers
--- a/build_helpers/install_ta-lib.sh
+++ b/build_helpers/install_ta-lib.sh
@ -12,9 +12,12 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then
  && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \
  && ./configure --prefix=${INSTALL_LOC}/ \
  && make -j$(nproc) \
-  && which sudo && sudo make install || make install \
-  && cd ..
+  && which sudo && sudo make install || make install
+  if [ -x "$(command -v apt-get)" ]; then
+    echo "Updating library path using ldconfig"
+    sudo ldconfig
+  fi
+  cd .. && rm -rf ./ta-lib/
 else
  echo "TA-lib already installed, skipping installation"
 fi
-#  && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \
--- a/docs/advanced-hyperopt.md
+++ b/docs/advanced-hyperopt.md
@ -80,7 +80,7 @@ To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_sp
 class MyAwesomeStrategy(IStrategy):
    class HyperOpt:
        # Define a custom stoploss space.
-        def stoploss_space(self):
+        def stoploss_space():
            return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')]
 ```

--- a/docs/configuration.md
+++ b/docs/configuration.md
@ -444,8 +444,8 @@ The possible values are: `gtc` (default), `fok` or `ioc`.
 ```

 !!! Warning
-    This is ongoing work. For now, it is supported only for binance.
-    Please don't change the default value unless you know what you are doing and have researched the impact of using different values.
+    This is ongoing work. For now, it is supported only for binance and kucoin.
+    Please don't change the default value unless you know what you are doing and have researched the impact of using different values for your particular exchange.

 ### Exchange configuration

--- a/docs/edge.md
+++ b/docs/edge.md
@ -3,7 +3,7 @@
 The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss.

 !!! Warning
-    WHen using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data.
+    When using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data.

 !!! Note
    `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file.
--- a/docs/exchanges.md
+++ b/docs/exchanges.md
@ -4,6 +4,8 @@ This page combines common gotchas and informations which are exchange-specific a

 ## Binance

+Binance supports [time_in_force](configuration.md#understand-order_time_in_force).
+
 !!! Tip "Stoploss on Exchange"
    Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it.

@ -113,8 +115,12 @@ Kucoin requires a passphrase for each api key, you will therefore need to add th
    "key": "your_exchange_key",
    "secret": "your_exchange_secret",
    "password": "your_exchange_api_key_password",
+    // ...
+}
 ```

+Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force).
+
 ### Kucoin Blacklists

 For Kucoin, please add `"KCS/<STAKE>"` to your blacklist to avoid issues.
@ -158,6 +164,8 @@ For example, to test the order type `FOK` with Kraken, and modify candle limit t
        "order_time_in_force": ["gtc", "fok"],
        "ohlcv_candle_limit": 200
        }
+    //...
+}
 ```

 !!! Warning
--- a/docs/requirements-docs.txt
+++ b/docs/requirements-docs.txt
@ -1,4 +1,4 @@
 mkdocs==1.2.2
-mkdocs-material==7.2.5
+mkdocs-material==7.2.6
 mdx_truly_sane_lists==1.2
 pymdown-extensions==8.2
--- a/freqtrade/init.py
+++ b/freqtrade/init.py
@ -22,7 +22,7 @@ if __version__ == 'develop':
        #  subprocess.check_output(
        #     ['git', 'log', '--format="%h"', '-n 1'],
        #     stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"')
-    except Exception:
+    except Exception:  # pragma: no cover
        # git not available, ignore
        try:
            # Try Fallback to freqtrade_commit file (created by CI while building docker image)
--- a/freqtrade/commands/build_config_commands.py
+++ b/freqtrade/commands/build_config_commands.py
@ -61,13 +61,13 @@ def ask_user_config() -> Dict[str, Any]:
            "type": "text",
            "name": "stake_currency",
            "message": "Please insert your stake currency:",
-            "default": 'BTC',
+            "default": 'USDT',
        },
        {
            "type": "text",
            "name": "stake_amount",
            "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):",
-            "default": "0.01",
+            "default": "100",
            "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val),
            "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"'
            if val == UNLIMITED_STAKE_AMOUNT
@ -105,6 +105,8 @@ def ask_user_config() -> Dict[str, Any]:
                "bittrex",
                "kraken",
                "ftx",
+                "kucoin",
+                "gateio",
                Separator(),
                "other",
            ],
@ -128,6 +130,12 @@ def ask_user_config() -> Dict[str, Any]:
            "message": "Insert Exchange Secret",
            "when": lambda x: not x['dry_run']
        },
+        {
+            "type": "password",
+            "name": "exchange_key_password",
+            "message": "Insert Exchange API Key password",
+            "when": lambda x: not x['dry_run'] and x['exchange_name'] == 'kucoin'
+        },
        {
            "type": "confirm",
            "name": "telegram",
--- a/freqtrade/exchange/binance.py
+++ b/freqtrade/exchange/binance.py
@ -18,6 +18,7 @@ class Binance(Exchange):
    _ft_has: Dict = {
        "stoploss_on_exchange": True,
        "order_time_in_force": ['gtc', 'fok', 'ioc'],
+        "time_in_force_parameter": "timeInForce",
        "ohlcv_candle_limit": 1000,
        "trades_pagination": "id",
        "trades_pagination_arg": "fromId",
--- a/freqtrade/exchange/exchange.py
+++ b/freqtrade/exchange/exchange.py
@ -54,12 +54,16 @@ class Exchange:
    # Parameters to add directly to buy/sell calls (like agreeing to trading agreement)
    _params: Dict = {}

+    # Additional headers - added to the ccxt object
+    _headers: Dict = {}
+
    # Dict to specify which options each exchange implements
    # This defines defaults, which can be selectively overridden by subclasses using _ft_has
    # or by specifying them in the configuration.
    _ft_has_default: Dict = {
        "stoploss_on_exchange": False,
        "order_time_in_force": ["gtc"],
+        "time_in_force_parameter": "timeInForce",
        "ohlcv_params": {},
        "ohlcv_candle_limit": 500,
        "ohlcv_partial_candle": True,
@ -169,7 +173,7 @@ class Exchange:
            asyncio.get_event_loop().run_until_complete(self._api_async.close())

    def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt,
-                   ccxt_kwargs: dict = None) -> ccxt.Exchange:
+                   ccxt_kwargs: Dict = {}) -> ccxt.Exchange:
        """
        Initialize ccxt with given config and return valid
        ccxt instance.
@ -188,6 +192,10 @@ class Exchange:
        }
        if ccxt_kwargs:
            logger.info('Applying additional ccxt config: %s', ccxt_kwargs)
+        if self._headers:
+            # Inject static headers after the above output to not confuse users.
+            ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs)
+        if ccxt_kwargs:
            ex_config.update(ccxt_kwargs)
        try:

@ -716,7 +724,8 @@ class Exchange:

        params = self._params.copy()
        if time_in_force != 'gtc' and ordertype != 'market':
-            params.update({'timeInForce': time_in_force})
+            param = self._ft_has.get('time_in_force_parameter', '')
+            params.update({param: time_in_force})

        try:
            # Set the precision for amount and price(rate) as accepted by the exchange
--- a/freqtrade/exchange/kucoin.py
+++ b/freqtrade/exchange/kucoin.py
@ -21,4 +21,6 @@ class Kucoin(Exchange):
    _ft_has: Dict = {
        "l2_limit_range": [20, 100],
        "l2_limit_range_required": False,
+        "order_time_in_force": ['gtc', 'fok', 'ioc'],
+        "time_in_force_parameter": "timeInForce",
    }
--- a/freqtrade/loggers.py
+++ b/freqtrade/loggers.py
@ -87,7 +87,7 @@ def setup_logging(config: Dict[str, Any]) -> None:
            # syslog config. The messages should be equal for this.
            handler_sl.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s'))
            logging.root.addHandler(handler_sl)
-        elif s[0] == 'journald':
+        elif s[0] == 'journald':  # pragma: no cover
            try:
                from systemd.journal import JournaldLogHandler
            except ImportError:
--- a/freqtrade/main.py
+++ b/freqtrade/main.py
@ -9,7 +9,7 @@ from typing import Any, List


 # check min. python version
-if sys.version_info < (3, 7):
+if sys.version_info < (3, 7):  # pragma: no cover
    sys.exit("Freqtrade requires Python version >= 3.7")

 from freqtrade.commands import Arguments
@ -46,7 +46,7 @@ def main(sysargv: List[str] = None) -> None:
                "`freqtrade --help` or `freqtrade <command> --help`."
            )

-    except SystemExit as e:
+    except SystemExit as e:  # pragma: no cover
        return_code = e
    except KeyboardInterrupt:
        logger.info('SIGINT received, aborting ...')
@ -60,5 +60,5 @@ def main(sysargv: List[str] = None) -> None:
        sys.exit(return_code)


-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
    main()
--- a/freqtrade/plugins/pairlist/pairlist_helpers.py
+++ b/freqtrade/plugins/pairlist/pairlist_helpers.py
@ -17,7 +17,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
    if keep_invalid:
        for pair_wc in wildcardpl:
            try:
-                comp = re.compile(pair_wc)
+                comp = re.compile(pair_wc, re.IGNORECASE)
                result_partial = [
                    pair for pair in available_pairs if re.fullmatch(comp, pair)
                ]
@ -33,7 +33,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str],
    else:
        for pair_wc in wildcardpl:
            try:
-                comp = re.compile(pair_wc)
+                comp = re.compile(pair_wc, re.IGNORECASE)
                result += [
                    pair for pair in available_pairs if re.fullmatch(comp, pair)
                ]
--- a/freqtrade/rpc/api_server/uvicorn_threaded.py
+++ b/freqtrade/rpc/api_server/uvicorn_threaded.py
@ -5,6 +5,20 @@ import time
 import uvicorn


+def asyncio_setup() -> None:  # pragma: no cover
+    # Set eventloop for win32 setups
+    # Reverts a change done in uvicorn 0.15.0 - which now sets the eventloop
+    # via policy.
+    import sys
+
+    if sys.version_info >= (3, 8) and sys.platform == "win32":
+        import asyncio
+        import selectors
+        selector = selectors.SelectSelector()
+        loop = asyncio.SelectorEventLoop(selector)
+        asyncio.set_event_loop(loop)
+
+
 class UvicornServer(uvicorn.Server):
    """
    Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742
@ -28,7 +42,7 @@ class UvicornServer(uvicorn.Server):
        try:
            import uvloop  # noqa
        except ImportError:  # pragma: no cover
-            from uvicorn.loops.asyncio import asyncio_setup
+
            asyncio_setup()
        else:
            asyncio.set_event_loop(uvloop.new_event_loop())
--- a/freqtrade/rpc/rpc.py
+++ b/freqtrade/rpc/rpc.py
@ -403,8 +403,11 @@ class RPC:
        # Doing the sum is not right - overall profit needs to be based on initial capital
        profit_all_ratio_sum = sum(profit_all_ratio) if profit_all_ratio else 0.0
        starting_balance = self._freqtrade.wallets.get_starting_balance()
-        profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance
-        profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance
+        profit_closed_ratio_fromstart = 0
+        profit_all_ratio_fromstart = 0
+        if starting_balance:
+            profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance
+            profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance

        profit_all_fiat = self._fiat_converter.convert_amount(
            profit_all_coin_sum,
--- a/freqtrade/templates/base_config.json.j2
+++ b/freqtrade/templates/base_config.json.j2
@ -1,3 +1,10 @@
+{%set volume_pairlist = '{
+            "method": "VolumePairList",
+            "number_assets": 20,
+            "sort_key": "quoteVolume",
+            "min_value": 0,
+            "refresh_period": 1800
+        }' %}
 {
    "max_open_trades": {{ max_open_trades }},
    "stake_currency": "{{ stake_currency }}",
@ -29,7 +36,7 @@
    },
    {{ exchange | indent(4) }},
    "pairlists": [
-        {"method": "StaticPairList"}
+        {{ '{"method": "StaticPairList"}' if exchange_name == 'bittrex' else  volume_pairlist }}
    ],
    "edge": {
        "enabled": false,
--- a/freqtrade/templates/subtemplates/exchange_binance.j2
+++ b/freqtrade/templates/subtemplates/exchange_binance.j2
@ -8,34 +8,8 @@
        "rateLimit": 200
    },
    "pair_whitelist": [
-        "ALGO/BTC",
-        "ATOM/BTC",
-        "BAT/BTC",
-        "BCH/BTC",
-        "BRD/BTC",
-        "EOS/BTC",
-        "ETH/BTC",
-        "IOTA/BTC",
-        "LINK/BTC",
-        "LTC/BTC",
-        "NEO/BTC",
-        "NXS/BTC",
-        "XMR/BTC",
-        "XRP/BTC",
-        "XTZ/BTC"
    ],
    "pair_blacklist": [
-        "BNB/BTC",
-        "BNB/BUSD",
-        "BNB/ETH",
-        "BNB/EUR",
-        "BNB/NGN",
-        "BNB/PAX",
-        "BNB/RUB",
-        "BNB/TRY",
-        "BNB/TUSD",
-        "BNB/USDC",
-        "BNB/USDS",
-        "BNB/USDT"
+        "BNB/.*"
    ]
 }
--- a/freqtrade/templates/subtemplates/exchange_bittrex.j2
+++ b/freqtrade/templates/subtemplates/exchange_bittrex.j2
@ -15,16 +15,6 @@
        "rateLimit": 500
    },
    "pair_whitelist": [
-        "ETH/BTC",
-        "LTC/BTC",
-        "ETC/BTC",
-        "DASH/BTC",
-        "ZEC/BTC",
-        "XLM/BTC",
-        "XRP/BTC",
-        "TRX/BTC",
-        "ADA/BTC",
-        "XMR/BTC"
    ],
    "pair_blacklist": [
    ]
--- a/freqtrade/templates/subtemplates/exchange_kraken.j2
+++ b/freqtrade/templates/subtemplates/exchange_kraken.j2
@ -7,28 +7,10 @@
    "ccxt_async_config": {
        "enableRateLimit": true,
        "rateLimit": 1000
+        // Enable the below for downoading data.
+        //"rateLimit": 3100
    },
    "pair_whitelist": [
-        "ADA/EUR",
-        "ATOM/EUR",
-        "BAT/EUR",
-        "BCH/EUR",
-        "BTC/EUR",
-        "DAI/EUR",
-        "DASH/EUR",
-        "EOS/EUR",
-        "ETC/EUR",
-        "ETH/EUR",
-        "LINK/EUR",
-        "LTC/EUR",
-        "QTUM/EUR",
-        "REP/EUR",
-        "WAVES/EUR",
-        "XLM/EUR",
-        "XMR/EUR",
-        "XRP/EUR",
-        "XTZ/EUR",
-        "ZEC/EUR"
    ],
    "pair_blacklist": [

--- a/freqtrade/templates/subtemplates/exchange_kucoin.j2
+++ b/freqtrade/templates/subtemplates/exchange_kucoin.j2
@ -0,0 +1,18 @@
+"exchange": {
+    "name": "{{ exchange_name | lower }}",
+    "key": "{{ exchange_key }}",
+    "secret": "{{ exchange_secret }}",
+    "password": "{{ exchange_key_password }}",
+    "ccxt_config": {
+        "enableRateLimit": true
+        "rateLimit": 200
+    },
+    "ccxt_async_config": {
+        "enableRateLimit": true,
+        "rateLimit": 200
+    },
+    "pair_whitelist": [
+    ],
+    "pair_blacklist": [
+    ]
+}
--- a/mkdocs.yml
+++ b/mkdocs.yml
@ -23,10 +23,10 @@ nav:
    - Hyperopt: hyperopt.md
    - Utility Sub-commands: utils.md
    - Plotting: plotting.md
+    - Exchange-specific Notes: exchanges.md
    - Data Analysis:
        - Jupyter Notebooks: data-analysis.md
        - Strategy analysis: strategy_analysis_example.md
-    - Exchange-specific Notes: exchanges.md
    - Advanced Topics:
        - Advanced Post-installation Tasks: advanced-setup.md
        - Edge Positioning: edge.md
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@ -8,7 +8,7 @@ flake8==3.9.2
 flake8-type-annotations==0.1.0
 flake8-tidy-imports==4.4.1
 mypy==0.910
-pytest==6.2.4
+pytest==6.2.5
 pytest-asyncio==0.15.1
 pytest-cov==2.12.1
 pytest-mock==3.6.1
--- a/requirements-plot.txt
+++ b/requirements-plot.txt
@ -1,5 +1,5 @@
 # Include all requirements to run the bot.
 -r requirements.txt

-plotly==5.3.0
+plotly==5.3.1

--- a/requirements.txt
+++ b/requirements.txt
@ -1,7 +1,7 @@
 numpy==1.21.2
 pandas==1.3.2

-ccxt==1.55.56
+ccxt==1.55.83
 # Pin cryptography for now due to rust build errors with piwheels
 cryptography==3.4.8
 aiohttp==3.7.4.post0
--- a/setup.sh
+++ b/setup.sh
@ -95,19 +95,7 @@ function install_talib() {
        return
    fi

-    cd build_helpers
-    tar zxvf ta-lib-0.4.0-src.tar.gz
-    cd ta-lib
-    sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h
-    ./configure --prefix=/usr/local
-    make
-    sudo make install
-    if [ -x "$(command -v apt-get)" ]; then
-        echo "Updating library path using ldconfig"
-        sudo ldconfig
-    fi
-    cd .. && rm -rf ./ta-lib/
-    cd ..
+    cd build_helpers && ./install_ta-lib.sh && cd ..
 }

 function install_mac_newer_python_dependencies() {    
--- a/tests/exchange/test_exchange.py
+++ b/tests/exchange/test_exchange.py
@ -108,6 +108,13 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog):
    assert hasattr(ex._api_async, 'TestKWARG')
    assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog)
    assert log_has(asynclogmsg, caplog)
+    # Test additional headers case
+    Exchange._headers = {'hello': 'world'}
+    ex = Exchange(conf)
+
+    assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog)
+    assert ex._api.headers == {'hello': 'world'}
+    Exchange._headers = {}


 def test_destroy(default_conf, mocker, caplog):
--- a/tests/strategy/test_interface.py
+++ b/tests/strategy/test_interface.py
@ -735,11 +735,16 @@ def test_auto_hyperopt_interface(default_conf):
    PairLocks.timeframe = default_conf['timeframe']
    strategy = StrategyResolver.load_strategy(default_conf)

+    with pytest.raises(OperationalException):
+        next(strategy.enumerate_parameters('deadBeef'))
+
    assert strategy.buy_rsi.value == strategy.buy_params['buy_rsi']
    # PlusDI is NOT in the buy-params, so default should be used
    assert strategy.buy_plusdi.value == 0.5
    assert strategy.sell_rsi.value == strategy.sell_params['sell_rsi']

+    assert repr(strategy.sell_rsi) == 'IntParameter(74)'
+
    # Parameter is disabled - so value from sell_param dict will NOT be used.
    assert strategy.sell_minusdi.value == 0.5
    all_params = strategy.detect_all_parameters()