From 097da448e2d297ace7f2158efb4fac6164168028 Mon Sep 17 00:00:00 2001 From: froggleston Date: Sat, 25 Sep 2021 15:48:42 +0100 Subject: [PATCH 01/50] Add CPU,RAM sysinfo support to the REST API to help with bot system monitoring --- freqtrade/rpc/api_server/api_v1.py | 4 ++++ freqtrade/rpc/rpc.py | 5 ++++- requirements.txt | 1 + scripts/rest_client.py | 6 ++++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 7e613f184..733fa7383 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -259,3 +259,7 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option 'pair_interval': pair_interval, } return result + +@router.get('/sysinfo', tags=['info']) +def sysinfo(rpc: RPC = Depends(get_rpc)): + return rpc._rpc_sysinfo() diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index f6599b429..9b0d4b0f7 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1,7 +1,7 @@ """ This module contains class to define a RPC communications """ -import logging +import logging, psutil from abc import abstractmethod from datetime import date, datetime, timedelta, timezone from math import isnan @@ -870,3 +870,6 @@ class RPC: 'subplots' not in self._freqtrade.strategy.plot_config): self._freqtrade.strategy.plot_config['subplots'] = {} return self._freqtrade.strategy.plot_config + + def _rpc_sysinfo(self) -> Dict[str, Any]: + return {"cpu_pct": psutil.cpu_percent(interval=1, percpu=True), "ram_pct": psutil.virtual_memory().percent} diff --git a/requirements.txt b/requirements.txt index d1d10dd1d..6a2cfec01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,7 @@ fastapi==0.68.1 uvicorn==0.15.0 pyjwt==2.1.0 aiofiles==0.7.0 +psutil==5.8.0 # Support for colorized terminal output colorama==0.4.4 diff --git a/scripts/rest_client.py b/scripts/rest_client.py index ece0a253e..52de3c534 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -334,6 +334,12 @@ class FtRestClient(): "timerange": timerange if timerange else '', }) + def sysinfo(self): + """Provides system information (CPU, RAM usage) + + :return: json object + """ + return self._get("sysinfo") def add_arguments(): parser = argparse.ArgumentParser() From e73f5ab4802823b6cfa2dba745d6218be0a7f97b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Oct 2021 09:48:19 +0200 Subject: [PATCH 02/50] Add test confirming #5652 --- tests/exchange/test_exchange.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 79b4a3ff5..691cf3c03 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -275,6 +275,7 @@ def test_amount_to_precision(default_conf, mocker, amount, precision_mode, preci (234.43, 4, 0.5, 234.5), (234.53, 4, 0.5, 235.0), (0.891534, 4, 0.0001, 0.8916), + (64968.89, 4, 0.01, 64968.89), ]) def test_price_to_precision(default_conf, mocker, price, precision_mode, precision, expected): @@ -293,7 +294,7 @@ def test_price_to_precision(default_conf, mocker, price, precision_mode, precisi PropertyMock(return_value=precision_mode)) pair = 'ETH/BTC' - assert pytest.approx(exchange.price_to_precision(pair, price)) == expected + assert exchange.price_to_precision(pair, price) == expected @pytest.mark.parametrize("price,precision_mode,precision,expected", [ From f5e5203388b6666664c5b9edd09683956d196478 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Oct 2021 09:48:50 +0200 Subject: [PATCH 03/50] Use "round" to 12 digits for TickSize mode Avoids float rounding problems, fix #5652 --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 2b9b08d70..e9d0316d2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -523,7 +523,7 @@ class Exchange: precision = self.markets[pair]['precision']['price'] missing = price % precision if missing != 0: - price = price - missing + precision + price = round(price - missing + precision, 10) else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) From 1c63d01cec878e363075e2f43d79296be2ab3d60 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Oct 2021 14:14:16 +0200 Subject: [PATCH 04/50] Prevent using market-orders on gateio GateIo does not support market orders on spot markets --- freqtrade/exchange/gateio.py | 8 ++++++++ tests/exchange/test_gateio.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tests/exchange/test_gateio.py diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index e6ee01c8a..018248a99 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -2,6 +2,7 @@ import logging from typing import Dict +from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange @@ -23,3 +24,10 @@ class Gateio(Exchange): } _headers = {'X-Gate-Channel-Id': 'freqtrade'} + + def validate_ordertypes(self, order_types: Dict) -> None: + super().validate_ordertypes(order_types) + + if any(v == 'market' for k, v in order_types.items()): + raise OperationalException( + f'Exchange {self.name} does not support market orders.') diff --git a/tests/exchange/test_gateio.py b/tests/exchange/test_gateio.py new file mode 100644 index 000000000..6f7862909 --- /dev/null +++ b/tests/exchange/test_gateio.py @@ -0,0 +1,28 @@ +import pytest + +from freqtrade.exceptions import OperationalException +from freqtrade.exchange import Gateio +from freqtrade.resolvers.exchange_resolver import ExchangeResolver + + +def test_validate_order_types_gateio(default_conf, mocker): + default_conf['exchange']['name'] = 'gateio' + mocker.patch('freqtrade.exchange.Exchange._init_ccxt') + mocker.patch('freqtrade.exchange.Exchange._load_markets', return_value={}) + mocker.patch('freqtrade.exchange.Exchange.validate_pairs') + mocker.patch('freqtrade.exchange.Exchange.validate_timeframes') + mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') + mocker.patch('freqtrade.exchange.Exchange.name', 'Bittrex') + exch = ExchangeResolver.load_exchange('gateio', default_conf, True) + assert isinstance(exch, Gateio) + + default_conf['order_types'] = { + 'buy': 'market', + 'sell': 'limit', + 'stoploss': 'market', + 'stoploss_on_exchange': False + } + + with pytest.raises(OperationalException, + match=r'Exchange .* does not support market orders.'): + ExchangeResolver.load_exchange('gateio', default_conf, True) From 0d9beaa3f36a23641d17ee3a7dd2e77933b39846 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:04 +0000 Subject: [PATCH 05/50] Bump filelock from 3.0.12 to 3.3.0 Bumps [filelock](https://github.com/tox-dev/py-filelock) from 3.0.12 to 3.3.0. - [Release notes](https://github.com/tox-dev/py-filelock/releases) - [Commits](https://github.com/tox-dev/py-filelock/compare/v3.0.12...3.3.0) --- updated-dependencies: - dependency-name: filelock dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 9feec80f1..b4067d1db 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -5,7 +5,7 @@ scipy==1.7.1 scikit-learn==0.24.2 scikit-optimize==0.8.1 -filelock==3.0.12 +filelock==3.3.0 joblib==1.0.1 psutil==5.8.0 progressbar2==3.53.3 From d220c55d405460de1eed9f5ae9e0dc214e6d8996 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:11 +0000 Subject: [PATCH 06/50] Bump pymdown-extensions from 8.2 to 9.0 Bumps [pymdown-extensions](https://github.com/facelessuser/pymdown-extensions) from 8.2 to 9.0. - [Release notes](https://github.com/facelessuser/pymdown-extensions/releases) - [Commits](https://github.com/facelessuser/pymdown-extensions/compare/8.2...9.0) --- updated-dependencies: - dependency-name: pymdown-extensions dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 9b7c12a43..67d2c9da8 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 mkdocs-material==7.3.0 mdx_truly_sane_lists==1.2 -pymdown-extensions==8.2 +pymdown-extensions==9.0 From ff45d52d497629340d9424728da87060fc369c64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:14 +0000 Subject: [PATCH 07/50] Bump types-filelock from 0.1.5 to 3.2.0 Bumps [types-filelock](https://github.com/python/typeshed) from 0.1.5 to 3.2.0. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-filelock dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2f03255a0..3d45247c1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -22,6 +22,6 @@ nbconvert==6.2.0 # mypy types types-cachetools==4.2.0 -types-filelock==0.1.5 +types-filelock==3.2.0 types-requests==2.25.9 types-tabulate==0.8.2 From 35c4a0a188c0a928600be63a177739d4fb1e42cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:17 +0000 Subject: [PATCH 08/50] Bump jsonschema from 3.2.0 to 4.0.1 Bumps [jsonschema](https://github.com/Julian/jsonschema) from 3.2.0 to 4.0.1. - [Release notes](https://github.com/Julian/jsonschema/releases) - [Changelog](https://github.com/Julian/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/Julian/jsonschema/compare/v3.2.0...v4.0.1) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index feeb4d942..370766f8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 wrapt==1.12.1 -jsonschema==3.2.0 +jsonschema==4.0.1 TA-Lib==0.4.21 technical==1.3.0 tabulate==0.8.9 From 0071d002b68c244716ba3010deb110bfbeaad2a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:22 +0000 Subject: [PATCH 09/50] Bump ccxt from 1.57.3 to 1.57.38 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.57.3 to 1.57.38. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.57.3...1.57.38) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index feeb4d942..5006d356f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.21.2 pandas==1.3.3 pandas-ta==0.3.14b -ccxt==1.57.3 +ccxt==1.57.38 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 From 2b41066ab7ccbe032586f231449f82704d37301f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 03:01:29 +0000 Subject: [PATCH 10/50] Bump pytest-cov from 2.12.1 to 3.0.0 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.12.1 to 3.0.0. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.12.1...v3.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 2f03255a0..109413e6e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ flake8-tidy-imports==4.4.1 mypy==0.910 pytest==6.2.5 pytest-asyncio==0.15.1 -pytest-cov==2.12.1 +pytest-cov==3.0.0 pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.9.3 From 949f4fbbbfdbae9987d212edae7b9a6a3c1db00c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 04:36:11 +0000 Subject: [PATCH 11/50] Bump types-cachetools from 4.2.0 to 4.2.2 Bumps [types-cachetools](https://github.com/python/typeshed) from 4.2.0 to 4.2.2. - [Release notes](https://github.com/python/typeshed/releases) - [Commits](https://github.com/python/typeshed/commits) --- updated-dependencies: - dependency-name: types-cachetools dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3d45247c1..858a46389 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -21,7 +21,7 @@ time-machine==2.4.0 nbconvert==6.2.0 # mypy types -types-cachetools==4.2.0 +types-cachetools==4.2.2 types-filelock==3.2.0 types-requests==2.25.9 types-tabulate==0.8.2 From f41fd4e88d7ca79294c40942a5fd181f574469d9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Oct 2021 04:40:24 +0000 Subject: [PATCH 12/50] Bump mkdocs-material from 7.3.0 to 7.3.1 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.3.0 to 7.3.1. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.3.0...7.3.1) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 67d2c9da8..bbbb240ba 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.3.0 +mkdocs-material==7.3.1 mdx_truly_sane_lists==1.2 pymdown-extensions==9.0 From f15922a16858247c6225d24dd69ae89ceb3e6034 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 4 Oct 2021 19:11:35 +0200 Subject: [PATCH 13/50] Fix custom_stoploss in strategy template closes #5658 --- .../templates/subtemplates/strategy_methods_advanced.j2 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 index 2df23f365..fb467ecaa 100644 --- a/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 +++ b/freqtrade/templates/subtemplates/strategy_methods_advanced.j2 @@ -32,8 +32,7 @@ def custom_stake_amount(self, pair: str, current_time: 'datetime', current_rate: use_custom_stoploss = True def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', - current_rate: float, current_profit: float, dataframe: DataFrame, - **kwargs) -> float: + current_rate: float, current_profit: float, **kwargs) -> float: """ Custom stoploss logic, returning the new distance relative to current_rate (as ratio). e.g. returning -0.05 would create a stoploss 5% below current_rate. @@ -44,14 +43,13 @@ def custom_stoploss(self, pair: str, trade: 'Trade', current_time: 'datetime', When not implemented by a strategy, returns the initial stoploss value Only called when use_custom_stoploss is set to True. - :param pair: Pair that's about to be sold. + :param pair: Pair that's currently analyzed :param trade: trade object. :param current_time: datetime object, containing the current datetime :param current_rate: Rate, calculated based on pricing settings in ask_strategy. :param current_profit: Current profit (as ratio), calculated based on current_rate. - :param dataframe: Analyzed dataframe for this pair. Can contain future data in backtesting. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return float: New stoploss value, relative to the currentrate + :return float: New stoploss value, relative to the current_rate """ return self.stoploss From 7f4baab420ce98c97deeedb69170c3129828c9b2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 4 Oct 2021 20:14:14 +0200 Subject: [PATCH 14/50] Remove explicit rateLimits, improve docs --- config_examples/config_binance.example.json | 4 +- config_examples/config_ftx.example.json | 7 +-- config_examples/config_full.example.json | 8 +-- config_examples/config_kraken.example.json | 4 +- docs/configuration.md | 39 --------------- docs/exchanges.md | 50 +++++++++++++++++++ docs/faq.md | 10 ++-- .../subtemplates/exchange_binance.j2 | 7 +-- .../subtemplates/exchange_generic.j2 | 6 +-- 9 files changed, 65 insertions(+), 70 deletions(-) diff --git a/config_examples/config_binance.example.json b/config_examples/config_binance.example.json index 938bc9342..d59ff96cb 100644 --- a/config_examples/config_binance.example.json +++ b/config_examples/config_binance.example.json @@ -28,10 +28,8 @@ "name": "binance", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_config": {"enableRateLimit": true}, + "ccxt_config": {}, "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 }, "pair_whitelist": [ "ALGO/BTC", diff --git a/config_examples/config_ftx.example.json b/config_examples/config_ftx.example.json index 48651f04c..4d9633cc0 100644 --- a/config_examples/config_ftx.example.json +++ b/config_examples/config_ftx.example.json @@ -28,11 +28,8 @@ "name": "ftx", "key": "your_exchange_key", "secret": "your_exchange_secret", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 50 - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ "BTC/USD", "ETH/USD", diff --git a/config_examples/config_full.example.json b/config_examples/config_full.example.json index c415d70b0..83b8a27d0 100644 --- a/config_examples/config_full.example.json +++ b/config_examples/config_full.example.json @@ -84,12 +84,8 @@ "key": "your_exchange_key", "secret": "your_exchange_secret", "password": "", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 500, - "aiohttp_trust_env": false - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ "ALGO/BTC", "ATOM/BTC", diff --git a/config_examples/config_kraken.example.json b/config_examples/config_kraken.example.json index bf3548568..32def895c 100644 --- a/config_examples/config_kraken.example.json +++ b/config_examples/config_kraken.example.json @@ -28,10 +28,8 @@ "name": "kraken", "key": "your_exchange_key", "secret": "your_exchange_key", - "ccxt_config": {"enableRateLimit": true}, + "ccxt_config": {}, "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 1000 }, "pair_whitelist": [ "ADA/EUR", diff --git a/docs/configuration.md b/docs/configuration.md index 6ccea4c73..bc8a40dcb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -447,45 +447,6 @@ The possible values are: `gtc` (default), `fok` or `ioc`. This is ongoing work. For now, it is supported only for binance and kucoin. Please don't change the default value unless you know what you are doing and have researched the impact of using different values for your particular exchange. -### Exchange configuration - -Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency -exchange markets and trading APIs. The complete up-to-date list can be found in the -[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). - However, the bot was tested by the development team with only Bittrex, Binance and Kraken, - so these are the only officially supported exchanges: - -- [Bittrex](https://bittrex.com/): "bittrex" -- [Binance](https://www.binance.com/): "binance" -- [Kraken](https://kraken.com/): "kraken" - -Feel free to test other exchanges and submit your PR to improve the bot. - -Some exchanges require special configuration, which can be found on the [Exchange-specific Notes](exchanges.md) documentation page. - -#### Sample exchange configuration - -A exchange configuration for "binance" would look as follows: - -```json -"exchange": { - "name": "binance", - "key": "your_exchange_key", - "secret": "your_exchange_secret", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 - }, -``` - -This configuration enables binance, as well as rate-limiting to avoid bans from the exchange. -`"rateLimit": 200` defines a wait-event of 0.2s between each call. This can also be completely disabled by setting `"enableRateLimit"` to false. - -!!! Note - Optimal settings for rate-limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. - We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. - ### What values can be used for fiat_display_currency? The `fiat_display_currency` configuration parameter sets the base currency to use for the diff --git a/docs/exchanges.md b/docs/exchanges.md index c0fbdc694..badaa484a 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -2,6 +2,56 @@ This page combines common gotchas and informations which are exchange-specific and most likely don't apply to other exchanges. +## Exchange configuration + +Freqtrade is based on [CCXT library](https://github.com/ccxt/ccxt) that supports over 100 cryptocurrency +exchange markets and trading APIs. The complete up-to-date list can be found in the +[CCXT repo homepage](https://github.com/ccxt/ccxt/tree/master/python). +However, the bot was tested by the development team with only a few exchanges. +A current list of these can be found in the "Home" section of this documentation. + +Feel free to test other exchanges and submit your feedback or PR to improve the bot or confirm exchanges that work flawlessly.. + +Some exchanges require special configuration, which can be found below. + +### Sample exchange configuration + +A exchange configuration for "binance" would look as follows: + +```json +"exchange": { + "name": "binance", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "ccxt_config": {}, + "ccxt_async_config": {}, + // ... +``` + +### Setting rate limits + +Usually, rate limits set by CCXT are reliable and work well. +In case of problems related to rate-limits (usually DDOS Exceptions in your logs), it's easy to change rateLimit settings to other values. + +```json +"exchange": { + "name": "kraken", + "key": "your_exchange_key", + "secret": "your_exchange_secret", + "ccxt_config": {"enableRateLimit": true}, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 3100 + }, +``` + +This configuration enables kraken, as well as rate-limiting to avoid bans from the exchange. +`"rateLimit": 3100` defines a wait-event of 0.2s between each call. This can also be completely disabled by setting `"enableRateLimit"` to false. + +!!! Note + Optimal settings for rate-limiting depend on the exchange and the size of the whitelist, so an ideal parameter will vary on many other settings. + We try to provide sensible defaults per exchange where possible, if you encounter bans please make sure that `"enableRateLimit"` is enabled and increase the `"rateLimit"` parameter step by step. + ## Binance Binance supports [time_in_force](configuration.md#understand-order_time_in_force). diff --git a/docs/faq.md b/docs/faq.md index 285625491..75c40a681 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -82,11 +82,11 @@ Currently known to happen for US Bittrex users. Read [the Bittrex section about restricted markets](exchanges.md#restricted-markets) for more information. -### I'm getting the "Exchange Bittrex does not support market orders." message and cannot run my strategy +### I'm getting the "Exchange XXX does not support market orders." message and cannot run my strategy -As the message says, Bittrex does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex). +As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex and Gate.io). -To fix it for Bittrex, redefine order types in the strategy to use "limit" instead of "market": +To fix this, redefine order types in the strategy to use "limit" instead of "market": ``` order_types = { @@ -136,6 +136,8 @@ On Windows, the `--logfile` option is also supported by Freqtrade and you can us > type \path\to\mylogfile.log | findstr "something" ``` +## Hyperopt module + ### Why does freqtrade not have GPU support? First of all, most indicator libraries don't have GPU support - as such, there would be little benefit for indicator calculations. @@ -152,8 +154,6 @@ The benefit of using GPU would therefore be pretty slim - and will not justify t There is however nothing preventing you from using GPU-enabled indicators within your strategy if you think you must have this - you will however probably be disappointed by the slim gain that will give you (compared to the complexity). -## Hyperopt module - ### How many epochs do I need to get a good Hyperopt result? Per default Hyperopt called without the `-e`/`--epochs` command line option will only diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index de58b6f72..dc2272119 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -2,11 +2,8 @@ "name": "{{ exchange_name | lower }}", "key": "{{ exchange_key }}", "secret": "{{ exchange_secret }}", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ ], "pair_blacklist": [ diff --git a/freqtrade/templates/subtemplates/exchange_generic.j2 b/freqtrade/templates/subtemplates/exchange_generic.j2 index ade9c2f28..08b11f365 100644 --- a/freqtrade/templates/subtemplates/exchange_generic.j2 +++ b/freqtrade/templates/subtemplates/exchange_generic.j2 @@ -2,10 +2,8 @@ "name": "{{ exchange_name | lower }}", "key": "{{ exchange_key }}", "secret": "{{ exchange_secret }}", - "ccxt_config": {"enableRateLimit": true}, - "ccxt_async_config": { - "enableRateLimit": true - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ ], From 92f8f231afe79eafa5079d494b56db71e7f30baf Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 4 Oct 2021 20:22:41 +0200 Subject: [PATCH 15/50] Remove ratelimit from kucoin template --- freqtrade/templates/subtemplates/exchange_kucoin.j2 | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/freqtrade/templates/subtemplates/exchange_kucoin.j2 b/freqtrade/templates/subtemplates/exchange_kucoin.j2 index 9882c51c7..b797dda41 100644 --- a/freqtrade/templates/subtemplates/exchange_kucoin.j2 +++ b/freqtrade/templates/subtemplates/exchange_kucoin.j2 @@ -3,14 +3,8 @@ "key": "{{ exchange_key }}", "secret": "{{ exchange_secret }}", "password": "{{ exchange_key_password }}", - "ccxt_config": { - "enableRateLimit": true, - "rateLimit": 200 - }, - "ccxt_async_config": { - "enableRateLimit": true, - "rateLimit": 200 - }, + "ccxt_config": {}, + "ccxt_async_config": {}, "pair_whitelist": [ ], "pair_blacklist": [ From 0db5c07314a02beb7dde0baf76a06fff458fd5c6 Mon Sep 17 00:00:00 2001 From: froggleston Date: Tue, 5 Oct 2021 00:10:39 +0100 Subject: [PATCH 16/50] Fix issues with sysinfo rpc/API code, add SysInfo api_schema --- freqtrade/rpc/api_server/api_schemas.py | 4 ++++ freqtrade/rpc/api_server/api_v1.py | 8 ++++---- freqtrade/rpc/rpc.py | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index 46187f571..b03400900 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -347,3 +347,7 @@ class BacktestResponse(BaseModel): trade_count: Optional[float] # TODO: Properly type backtestresult... backtest_result: Optional[Dict[str, Any]] + +class SysInfo(BaseModel): + cpu_pct: float + ram_pct: float diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 733fa7383..d52e8c10d 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -18,7 +18,7 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac OpenTradeSchema, PairHistory, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, - StrategyResponse, Version, WhitelistResponse) + StrategyResponse, SysInfo, Version, WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -260,6 +260,6 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option } return result -@router.get('/sysinfo', tags=['info']) -def sysinfo(rpc: RPC = Depends(get_rpc)): - return rpc._rpc_sysinfo() +@router.get('/sysinfo', response_model=SysInfo, tags=['info']) +def sysinfo(): + return RPC._rpc_sysinfo() diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 9b0d4b0f7..699e3b384 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -871,5 +871,6 @@ class RPC: self._freqtrade.strategy.plot_config['subplots'] = {} return self._freqtrade.strategy.plot_config - def _rpc_sysinfo(self) -> Dict[str, Any]: + @staticmethod + def _rpc_sysinfo() -> Dict[str, Any]: return {"cpu_pct": psutil.cpu_percent(interval=1, percpu=True), "ram_pct": psutil.virtual_memory().percent} From 949d616082c49be11211a396de676077ad741c8d Mon Sep 17 00:00:00 2001 From: jonny07 Date: Tue, 5 Oct 2021 21:33:15 +0200 Subject: [PATCH 17/50] Update docker_quickstart.md Got help in the discord chat to get the UI running, I think most people will need this... --- docs/docker_quickstart.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 27a9091b1..0fe69933a 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -70,6 +70,40 @@ docker-compose up -d !!! Warning "Default configuration" While the configuration generated will be mostly functional, you will still need to verify that all options correspond to what you want (like Pricing, pairlist, ...) before starting the bot. +#### Acessing the UI + +Uncommend the 2 lines below and add your IP adress in the following format (like 192.168.2.67:8080:8080) to the ft_userdata/docker-compose.yml: +'''bash + ports: + - "yourIPadress:8080:8080" +''' +Your ft_userdata/user_data/config.json should look like: +'''bash +api_server": { + "enabled": true, + "listen_ip_address": "0.0.0.0", + "listen_port": 8080, + "verbosity": "error", + "enable_openapi": false, + "jwt_secret_key": "****", + "CORS_origins": [], + "username": "****", + "password": "****" + }, +''' +instead of "****" you will have your data in. +Then rebuild your docker file: +Linux: +'''bash +sudo docker-compose down && sudo docker-compose pull && sudo docker-compose build && sudo docker-compose up -d +''' +Windows: +'''bash +docker-compose down && docker-compose pull && docker-compose build && docker-compose up -d +''' + +You can now access the UI by typing yourIPadress:8080 in your browser. + #### Monitoring the bot You can check for running instances with `docker-compose ps`. From c0d01dbc26a1552562ca339a92111f6193e7a02c Mon Sep 17 00:00:00 2001 From: sid Date: Wed, 6 Oct 2021 13:24:27 +0530 Subject: [PATCH 18/50] add max_drawdown loss --- .../optimize/hyperopt_loss_max_drawdown.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 freqtrade/optimize/hyperopt_loss_max_drawdown.py diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss_max_drawdown.py new file mode 100644 index 000000000..e6f73e04a --- /dev/null +++ b/freqtrade/optimize/hyperopt_loss_max_drawdown.py @@ -0,0 +1,42 @@ +""" +MaxDrawDownHyperOptLoss + +This module defines the alternative HyperOptLoss class which can be used for +Hyperoptimization. +""" +from datetime import datetime +from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.optimize.hyperopt import IHyperOptLoss + +from pandas import DataFrame + + +class MaxDrawDownHyperOptLoss(IHyperOptLoss): + + """ + Defines the loss function for hyperopt. + + This implementation optimizes for max draw down and profit + Less max drawdown more profit -> Lower return value + """ + + @staticmethod + def hyperopt_loss_function(results: DataFrame, trade_count: int, + min_date: datetime, max_date: datetime, + *args, **kwargs) -> float: + + """ + Objective function. + + Uses profit ratio weighted max_drawdown when drawdown is available. + Otherwise directly optimizes profit ratio. + """ + total_profit = results['profit_ratio'].sum() + try: + max_drawdown = calculate_max_drawdown(results) + except ValueError: + # No losing trade, therefore no drawdown. + return -total_profit + max_drawdown_rev = 1 / max_drawdown[0] + ret = max_drawdown_rev * total_profit + return -ret \ No newline at end of file From 6ba46b38bdd434ddd65b065c6393beb0b29aa492 Mon Sep 17 00:00:00 2001 From: sid Date: Wed, 6 Oct 2021 13:46:05 +0530 Subject: [PATCH 19/50] fix formatting --- freqtrade/optimize/hyperopt_loss_max_drawdown.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss_max_drawdown.py index e6f73e04a..4fa32c00e 100644 --- a/freqtrade/optimize/hyperopt_loss_max_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss_max_drawdown.py @@ -5,11 +5,12 @@ This module defines the alternative HyperOptLoss class which can be used for Hyperoptimization. """ from datetime import datetime -from freqtrade.data.btanalysis import calculate_max_drawdown -from freqtrade.optimize.hyperopt import IHyperOptLoss from pandas import DataFrame +from freqtrade.data.btanalysis import calculate_max_drawdown +from freqtrade.optimize.hyperopt import IHyperOptLoss + class MaxDrawDownHyperOptLoss(IHyperOptLoss): @@ -31,7 +32,7 @@ class MaxDrawDownHyperOptLoss(IHyperOptLoss): Uses profit ratio weighted max_drawdown when drawdown is available. Otherwise directly optimizes profit ratio. """ - total_profit = results['profit_ratio'].sum() + total_profit = results['profit_ratio'].sum() try: max_drawdown = calculate_max_drawdown(results) except ValueError: @@ -39,4 +40,4 @@ class MaxDrawDownHyperOptLoss(IHyperOptLoss): return -total_profit max_drawdown_rev = 1 / max_drawdown[0] ret = max_drawdown_rev * total_profit - return -ret \ No newline at end of file + return -ret From 57ef25789e3db1dd59de1e76096bf730fefdf0d7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Oct 2021 19:36:28 +0200 Subject: [PATCH 20/50] Fix style errors --- freqtrade/rpc/api_server/api_schemas.py | 1 + freqtrade/rpc/api_server/api_v1.py | 4 +++- freqtrade/rpc/rpc.py | 8 ++++++-- scripts/rest_client.py | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index b03400900..bde6af35b 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -348,6 +348,7 @@ class BacktestResponse(BaseModel): # TODO: Properly type backtestresult... backtest_result: Optional[Dict[str, Any]] + class SysInfo(BaseModel): cpu_pct: float ram_pct: float diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index d52e8c10d..06230a7db 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -18,7 +18,8 @@ from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, Blac OpenTradeSchema, PairHistory, PerformanceEntry, Ping, PlotConfig, Profit, ResultMsg, ShowConfig, Stats, StatusMsg, StrategyListResponse, - StrategyResponse, SysInfo, Version, WhitelistResponse) + StrategyResponse, SysInfo, Version, + WhitelistResponse) from freqtrade.rpc.api_server.deps import get_config, get_rpc, get_rpc_optional from freqtrade.rpc.rpc import RPCException @@ -260,6 +261,7 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option } return result + @router.get('/sysinfo', response_model=SysInfo, tags=['info']) def sysinfo(): return RPC._rpc_sysinfo() diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 699e3b384..d0858350c 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -1,13 +1,14 @@ """ This module contains class to define a RPC communications """ -import logging, psutil +import logging from abc import abstractmethod from datetime import date, datetime, timedelta, timezone from math import isnan from typing import Any, Dict, List, Optional, Tuple, Union import arrow +import psutil from numpy import NAN, inf, int64, mean from pandas import DataFrame @@ -873,4 +874,7 @@ class RPC: @staticmethod def _rpc_sysinfo() -> Dict[str, Any]: - return {"cpu_pct": psutil.cpu_percent(interval=1, percpu=True), "ram_pct": psutil.virtual_memory().percent} + return { + "cpu_pct": psutil.cpu_percent(interval=1, percpu=True), + "ram_pct": psutil.virtual_memory().percent + } diff --git a/scripts/rest_client.py b/scripts/rest_client.py index 52de3c534..ac3b6defe 100755 --- a/scripts/rest_client.py +++ b/scripts/rest_client.py @@ -341,6 +341,7 @@ class FtRestClient(): """ return self._get("sysinfo") + def add_arguments(): parser = argparse.ArgumentParser() parser.add_argument("command", From 992cef56e653844c4b8613c683db81057959f569 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Oct 2021 19:36:51 +0200 Subject: [PATCH 21/50] Add test for sysinfo endpoint --- tests/rpc/test_rpc_apiserver.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 7c98b2df7..117b1fa49 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1271,6 +1271,16 @@ def test_list_available_pairs(botclient): assert len(rc.json()['pair_interval']) == 1 +def test_sysinfo(botclient): + ftbot, client = botclient + + rc = client_get(client, f"{BASE_URI}/sysinfo") + assert_response(rc) + result = rc.json() + assert 'cpu_pct' in result + assert 'ram_pct' in result + + def test_api_backtesting(botclient, mocker, fee, caplog): ftbot, client = botclient mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) From 65d4df938df14881729a63bf2c58ace1aca57d22 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Oct 2021 20:09:08 +0200 Subject: [PATCH 22/50] Improve docker port api --- docker-compose.yml | 6 +++--- docs/docker_quickstart.md | 43 ++++++++++++++++++++------------------- docs/rest-api.md | 2 +- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 80e194ab2..445fbaea0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,10 +15,10 @@ services: volumes: - "./user_data:/freqtrade/user_data" # Expose api on port 8080 (localhost only) - # Please read the https://www.freqtrade.io/en/latest/rest-api/ documentation + # Please read the https://www.freqtrade.io/en/stable/rest-api/ documentation # before enabling this. - # ports: - # - "127.0.0.1:8080:8080" + ports: + - "127.0.0.1:8080:8080" # Default command used when running `docker compose up` command: > trade diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index 27a9091b1..d5bec54c9 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -148,27 +148,9 @@ You'll then also need to modify the `docker-compose.yml` file and uncomment the dockerfile: "./Dockerfile." ``` -You can then run `docker-compose build` to build the docker image, and run it using the commands described above. +You can then run `docker-compose build --pull` to build the docker image, and run it using the commands described above. -### Troubleshooting - -#### Docker on Windows - -* Error: `"Timestamp for this request is outside of the recvWindow."` - * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past. - To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so). - A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler. - ``` - taskkill /IM "Docker Desktop.exe" /F - wsl --shutdown - start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe" - ``` - -!!! Warning - Due to the above, we do not recommend the usage of docker on windows for production setups, but only for experimentation, datadownload and backtesting. - Best use a linux-VPS for running freqtrade reliably. - -## Plotting with docker-compose +### Plotting with docker-compose Commands `freqtrade plot-profit` and `freqtrade plot-dataframe` ([Documentation](plotting.md)) are available by changing the image to `*_plot` in your docker-compose.yml file. You can then use these commands as follows: @@ -179,7 +161,7 @@ docker-compose run --rm freqtrade plot-dataframe --strategy AwesomeStrategy -p B The output will be stored in the `user_data/plot` directory, and can be opened with any modern browser. -## Data analysis using docker compose +### Data analysis using docker compose Freqtrade provides a docker-compose file which starts up a jupyter lab server. You can run this server using the following command: @@ -196,3 +178,22 @@ Since part of this image is built on your machine, it is recommended to rebuild ``` bash docker-compose -f docker/docker-compose-jupyter.yml build --no-cache ``` + +## Troubleshooting + +### Docker on Windows + +* Error: `"Timestamp for this request is outside of the recvWindow."` + * The market api requests require a synchronized clock but the time in the docker container shifts a bit over time into the past. + To fix this issue temporarily you need to run `wsl --shutdown` and restart docker again (a popup on windows 10 will ask you to do so). + A permanent solution is either to host the docker container on a linux host or restart the wsl from time to time with the scheduler. + + ``` bash + taskkill /IM "Docker Desktop.exe" /F + wsl --shutdown + start "" "C:\Program Files\Docker\Docker\Docker Desktop.exe" + ``` + +!!! Warning + Due to the above, we do not recommend the usage of docker on windows for production setups, but only for experimentation, datadownload and backtesting. + Best use a linux-VPS for running freqtrade reliably. diff --git a/docs/rest-api.md b/docs/rest-api.md index b9b2b29be..b4992e047 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -78,7 +78,7 @@ If you run your bot using docker, you'll need to have the bot listen to incoming }, ``` -Uncomment the following from your docker-compose file: +Make sure that the following 2 lines are available in your docker-compose file: ```yml ports: From 526bdaa2dc04cd84dbea35c72482697cc865080e Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Oct 2021 20:14:59 +0200 Subject: [PATCH 23/50] Recommend using 0.0.0.0 as listen address for docker --- freqtrade/commands/build_config_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index faa8a98f4..34ae35aff 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -163,7 +163,8 @@ def ask_user_config() -> Dict[str, Any]: { "type": "text", "name": "api_server_listen_addr", - "message": "Insert Api server Listen Address (best left untouched default!)", + "message": ("Insert Api server Listen Address (0.0.0.0 for docker, " + "otherwise best left untouched)"), "default": "127.0.0.1", "when": lambda x: x['api_server'] }, From 46c320513aa0b825b370034feba9d6e9f29af312 Mon Sep 17 00:00:00 2001 From: sid Date: Thu, 7 Oct 2021 08:07:07 +0530 Subject: [PATCH 24/50] use profit_abs --- freqtrade/optimize/hyperopt_loss_max_drawdown.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss_max_drawdown.py index 4fa32c00e..6777fb2e8 100644 --- a/freqtrade/optimize/hyperopt_loss_max_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss_max_drawdown.py @@ -32,9 +32,9 @@ class MaxDrawDownHyperOptLoss(IHyperOptLoss): Uses profit ratio weighted max_drawdown when drawdown is available. Otherwise directly optimizes profit ratio. """ - total_profit = results['profit_ratio'].sum() + total_profit = results['profit_abs'].sum() try: - max_drawdown = calculate_max_drawdown(results) + max_drawdown = calculate_max_drawdown(results, value_col='profit_abs') except ValueError: # No losing trade, therefore no drawdown. return -total_profit From 29863ad2bf7d364e0bc17dc5c3506e24f20b31b4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Oct 2021 06:51:29 +0200 Subject: [PATCH 25/50] Fix error when ask_last_balance is not set closes #5181 --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e9d0316d2..7cc436430 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1058,7 +1058,7 @@ class Exchange: ticker_rate = ticker[conf_strategy['price_side']] if ticker['last'] and ticker_rate: if side == 'buy' and ticker_rate > ticker['last']: - balance = conf_strategy['ask_last_balance'] + balance = conf_strategy.get('ask_last_balance', 0.0) ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) elif side == 'sell' and ticker_rate < ticker['last']: balance = conf_strategy.get('bid_last_balance', 0.0) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 691cf3c03..8cb494edb 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1832,6 +1832,7 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): ('ask', 20, 19, 10, 0.3, 17), # Between ask and last ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask + ('ask', 20, 19, 10, None, 20), # ask_last_balance missing ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask ('ask', 4, 5, None, 1, 4), # last not available - uses ask @@ -1842,6 +1843,7 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): ('bid', 21, 20, 10, 0.7, 13), # Between bid and last ('bid', 21, 20, 10, 0.3, 17), # Between bid and last ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid + ('bid', 21, 20, 10, None, 20), # ask_last_balance missing ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid @@ -1851,7 +1853,10 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, last, last_ab, expected) -> None: caplog.set_level(logging.DEBUG) - default_conf['bid_strategy']['ask_last_balance'] = last_ab + if last_ab is None: + del default_conf['bid_strategy']['ask_last_balance'] + else: + default_conf['bid_strategy']['ask_last_balance'] = last_ab default_conf['bid_strategy']['price_side'] = side exchange = get_patched_exchange(mocker, default_conf) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', @@ -1876,6 +1881,7 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), + ('bid', 0.003, 0.002, 0.005, None, 0.002), ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat @@ -1886,6 +1892,7 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), + ('ask', 0.006, 1.0, 11.0, None, 0.006), ]) def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, last, last_ab, expected) -> None: From 45b7a0c8377a5493496abafa70e48b0872d0b4b5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Oct 2021 07:12:45 +0200 Subject: [PATCH 26/50] Add Test and docs for MaxDrawDownHyperOptLoss --- docs/hyperopt.md | 18 ++++++++++-------- freqtrade/constants.py | 3 ++- tests/optimize/test_hyperoptloss.py | 5 +++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 09d43939a..45e0d444d 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -60,7 +60,7 @@ optional arguments: Specify what timerange of data to use. --data-format-ohlcv {json,jsongz,hdf5} Storage format for downloaded candle (OHLCV) data. - (default: `None`). + (default: `json`). --max-open-trades INT Override the value of the `max_open_trades` configuration setting. @@ -114,7 +114,8 @@ optional arguments: Hyperopt-loss-functions are: ShortTradeDurHyperOptLoss, OnlyProfitHyperOptLoss, SharpeHyperOptLoss, SharpeHyperOptLossDaily, - SortinoHyperOptLoss, SortinoHyperOptLossDaily + SortinoHyperOptLoss, SortinoHyperOptLossDaily, + MaxDrawDownHyperOptLoss --disable-param-export Disable automatic hyperopt parameter export. @@ -512,12 +513,13 @@ This class should be in its own file within the `user_data/hyperopts/` directory Currently, the following loss functions are builtin: -* `ShortTradeDurHyperOptLoss` (default legacy Freqtrade hyperoptimization loss function) - Mostly for short trade duration and avoiding losses. -* `OnlyProfitHyperOptLoss` (which takes only amount of profit into consideration) -* `SharpeHyperOptLoss` (optimizes Sharpe Ratio calculated on trade returns relative to standard deviation) -* `SharpeHyperOptLossDaily` (optimizes Sharpe Ratio calculated on **daily** trade returns relative to standard deviation) -* `SortinoHyperOptLoss` (optimizes Sortino Ratio calculated on trade returns relative to **downside** standard deviation) -* `SortinoHyperOptLossDaily` (optimizes Sortino Ratio calculated on **daily** trade returns relative to **downside** standard deviation) +* `ShortTradeDurHyperOptLoss` - (default legacy Freqtrade hyperoptimization loss function) - Mostly for short trade duration and avoiding losses. +* `OnlyProfitHyperOptLoss` - takes only amount of profit into consideration. +* `SharpeHyperOptLoss` - optimizes Sharpe Ratio calculated on trade returns relative to standard deviation. +* `SharpeHyperOptLossDaily` - optimizes Sharpe Ratio calculated on **daily** trade returns relative to standard deviation. +* `SortinoHyperOptLoss` - optimizes Sortino Ratio calculated on trade returns relative to **downside** standard deviation. +* `SortinoHyperOptLossDaily` - optimizes Sortino Ratio calculated on **daily** trade returns relative to **downside** standard deviation. +* `MaxDrawDownHyperOptLoss` - Optimizes Maximum drawdown. Creation of a custom loss function is covered in the [Advanced Hyperopt](advanced-hyperopt.md) part of the documentation. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index fca319a0f..c6b8f0e62 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -24,7 +24,8 @@ ORDERTYPE_POSSIBILITIES = ['limit', 'market'] ORDERTIF_POSSIBILITIES = ['gtc', 'fok', 'ioc'] HYPEROPT_LOSS_BUILTIN = ['ShortTradeDurHyperOptLoss', 'OnlyProfitHyperOptLoss', 'SharpeHyperOptLoss', 'SharpeHyperOptLossDaily', - 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily'] + 'SortinoHyperOptLoss', 'SortinoHyperOptLossDaily', + 'MaxDrawDownHyperOptLoss'] AVAILABLE_PAIRLISTS = ['StaticPairList', 'VolumePairList', 'AgeFilter', 'OffsetFilter', 'PerformanceFilter', 'PrecisionFilter', 'PriceFilter', 'RangeStabilityFilter', diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index 923e3fc32..a39190934 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -84,13 +84,14 @@ def test_loss_calculation_has_limited_profit(hyperopt_conf, hyperopt_results) -> "SortinoHyperOptLossDaily", "SharpeHyperOptLoss", "SharpeHyperOptLossDaily", + "MaxDrawDownHyperOptLoss", ]) def test_loss_functions_better_profits(default_conf, hyperopt_results, lossfunction) -> None: results_over = hyperopt_results.copy() - results_over['profit_abs'] = hyperopt_results['profit_abs'] * 2 + results_over['profit_abs'] = hyperopt_results['profit_abs'] * 2 + 0.2 results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 results_under = hyperopt_results.copy() - results_under['profit_abs'] = hyperopt_results['profit_abs'] / 2 + results_under['profit_abs'] = hyperopt_results['profit_abs'] / 2 - 0.2 results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 default_conf.update({'hyperopt_loss': lossfunction}) From a1be6124f221d6e3bd294c376a76848ff3e6d197 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Oct 2021 07:15:09 +0200 Subject: [PATCH 27/50] Don't set bid_last_balance if None in tests part of #5681 --- tests/exchange/test_exchange.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 8cb494edb..e3369182d 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1899,7 +1899,8 @@ def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, caplog.set_level(logging.DEBUG) default_conf['ask_strategy']['price_side'] = side - default_conf['ask_strategy']['bid_last_balance'] = last_ab + if last_ab is not None: + default_conf['ask_strategy']['bid_last_balance'] = last_ab mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', return_value={'ask': ask, 'bid': bid, 'last': last}) pair = "ETH/BTC" From f07eeddda0e912e44467bff42b6c08f125902ae1 Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Thu, 7 Oct 2021 12:04:42 +0100 Subject: [PATCH 28/50] Update api_schemas.py Fix api schema for cpu_pct float List. --- freqtrade/rpc/api_server/api_schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/api_schemas.py b/freqtrade/rpc/api_server/api_schemas.py index bde6af35b..e9985c3c6 100644 --- a/freqtrade/rpc/api_server/api_schemas.py +++ b/freqtrade/rpc/api_server/api_schemas.py @@ -350,5 +350,5 @@ class BacktestResponse(BaseModel): class SysInfo(BaseModel): - cpu_pct: float + cpu_pct: List[float] ram_pct: float From 1327c21d0103eee5e040e03e598662c993bcfac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E0=AE=AE=E0=AE=A9=E0=AF=8B=E0=AE=9C=E0=AF=8D=E0=AE=95?= =?UTF-8?q?=E0=AF=81=E0=AE=AE=E0=AE=BE=E0=AE=B0=E0=AF=8D=20=E0=AE=AA?= =?UTF-8?q?=E0=AE=B4=E0=AE=A9=E0=AE=BF=E0=AE=9A=E0=AF=8D=E0=AE=9A=E0=AE=BE?= =?UTF-8?q?=E0=AE=AE=E0=AE=BF?= Date: Thu, 7 Oct 2021 19:12:09 +0530 Subject: [PATCH 29/50] Update README.md --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 01effd7bc..79763983d 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,7 @@ Please find the complete documentation on our [website](https://www.freqtrade.io Freqtrade provides a Linux/macOS script to install all dependencies and help you to configure the bot. ```bash -git clone -b develop https://github.com/freqtrade/freqtrade.git -cd freqtrade -./setup.sh --install +pip install freqtrade ``` For any other type of installation please refer to [Installation doc](https://www.freqtrade.io/en/latest/installation/). From 482f4418c68c2b99635a7af3cc65a14c1022e031 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Oct 2021 14:36:52 +0200 Subject: [PATCH 30/50] Clarify "required candle" message --- freqtrade/exchange/exchange.py | 62 ++++++++++++++-------------------- 1 file changed, 25 insertions(+), 37 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7cc436430..b6cfb8d8b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -26,9 +26,9 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, - EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, - remove_credentials, retrier, retrier_async) -from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 + EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, + retrier_async) +from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -54,16 +54,12 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} - # Additional headers - added to the ccxt object - _headers: Dict = {} - # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has # or by specifying them in the configuration. _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], - "time_in_force_parameter": "timeInForce", "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, @@ -104,7 +100,6 @@ class Exchange: # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} - remove_credentials(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') @@ -174,7 +169,7 @@ class Exchange: asyncio.get_event_loop().run_until_complete(self._api_async.close()) def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, - ccxt_kwargs: Dict = {}) -> ccxt.Exchange: + ccxt_kwargs: dict = None) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -193,10 +188,6 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) - if self._headers: - # Inject static headers after the above output to not confuse users. - ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) - if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: @@ -480,7 +471,7 @@ class Exchange: if startup_candles + 5 > candle_limit: raise OperationalException( f"This strategy requires {startup_candles} candles to start. " - f"{self.name} only provides {candle_limit} for {timeframe}.") + f"{self.name} only provides {candle_limit - 5} for {timeframe}.") def exchange_has(self, endpoint: str) -> bool: """ @@ -523,7 +514,7 @@ class Exchange: precision = self.markets[pair]['precision']['price'] missing = price % precision if missing != 0: - price = round(price - missing + precision, 10) + price = price - missing + precision else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) @@ -725,8 +716,7 @@ class Exchange: params = self._params.copy() if time_in_force != 'gtc' and ordertype != 'market': - param = self._ft_has.get('time_in_force_parameter', '') - params.update({param: time_in_force}) + params.update({'timeInForce': time_in_force}) try: # Set the precision for amount and price(rate) as accepted by the exchange @@ -1058,7 +1048,7 @@ class Exchange: ticker_rate = ticker[conf_strategy['price_side']] if ticker['last'] and ticker_rate: if side == 'buy' and ticker_rate > ticker['last']: - balance = conf_strategy.get('ask_last_balance', 0.0) + balance = conf_strategy['ask_last_balance'] ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) elif side == 'sell' and ticker_rate < ticker['last']: balance = conf_strategy.get('bid_last_balance', 0.0) @@ -1195,7 +1185,7 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False) -> List: + since_ms: int) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1207,7 +1197,7 @@ class Exchange: """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms, is_new_pair=is_new_pair)) + since_ms=since_ms)) def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1222,12 +1212,11 @@ class Exchange: return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool - ) -> List: + async def _async_get_historic_ohlcv(self, pair: str, + timeframe: str, + since_ms: int) -> List: """ Download historic ohlcv - :param is_new_pair: used by binance subclass to allow "fast" new pair downloading """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1240,22 +1229,21 @@ class Exchange: pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - data: List = [] - # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling - for input_coro in chunks(input_coroutines, 100): + results = await asyncio.gather(*input_coroutines, return_exceptions=True) - results = await asyncio.gather(*input_coro, return_exceptions=True) - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: - data.extend(new_data) + # Combine gathered results + data: List = [] + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - logger.info(f"Downloaded data for {pair} with length {len(data)}.") + logger.info("Downloaded data for %s with length %s.", pair, len(data)) return data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, From 11ec1d9b062ae7a063d8b6549cdfd5e468047d63 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Oct 2021 20:22:07 +0200 Subject: [PATCH 31/50] Revert previous commit --- freqtrade/exchange/exchange.py | 60 ++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b6cfb8d8b..4143b79a5 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -26,9 +26,9 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, - EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, - retrier_async) -from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 + EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, + remove_credentials, retrier, retrier_async) +from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -54,12 +54,16 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} + # Additional headers - added to the ccxt object + _headers: Dict = {} + # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has # or by specifying them in the configuration. _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "time_in_force_parameter": "timeInForce", "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, @@ -100,6 +104,7 @@ class Exchange: # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} + remove_credentials(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') @@ -169,7 +174,7 @@ class Exchange: asyncio.get_event_loop().run_until_complete(self._api_async.close()) def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, - ccxt_kwargs: dict = None) -> ccxt.Exchange: + ccxt_kwargs: Dict = {}) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -188,6 +193,10 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) + if self._headers: + # Inject static headers after the above output to not confuse users. + ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) + if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: @@ -514,7 +523,7 @@ class Exchange: precision = self.markets[pair]['precision']['price'] missing = price % precision if missing != 0: - price = price - missing + precision + price = round(price - missing + precision, 10) else: symbol_prec = self.markets[pair]['precision']['price'] big_price = price * pow(10, symbol_prec) @@ -716,7 +725,8 @@ class Exchange: params = self._params.copy() if time_in_force != 'gtc' and ordertype != 'market': - params.update({'timeInForce': time_in_force}) + param = self._ft_has.get('time_in_force_parameter', '') + params.update({param: time_in_force}) try: # Set the precision for amount and price(rate) as accepted by the exchange @@ -1048,7 +1058,7 @@ class Exchange: ticker_rate = ticker[conf_strategy['price_side']] if ticker['last'] and ticker_rate: if side == 'buy' and ticker_rate > ticker['last']: - balance = conf_strategy['ask_last_balance'] + balance = conf_strategy.get('ask_last_balance', 0.0) ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) elif side == 'sell' and ticker_rate < ticker['last']: balance = conf_strategy.get('bid_last_balance', 0.0) @@ -1185,7 +1195,7 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int) -> List: + since_ms: int, is_new_pair: bool = False) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1197,7 +1207,7 @@ class Exchange: """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms)) + since_ms=since_ms, is_new_pair=is_new_pair)) def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1212,11 +1222,12 @@ class Exchange: return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, - timeframe: str, - since_ms: int) -> List: + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: """ Download historic ohlcv + :param is_new_pair: used by binance subclass to allow "fast" new pair downloading """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1229,21 +1240,22 @@ class Exchange: pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - results = await asyncio.gather(*input_coroutines, return_exceptions=True) - - # Combine gathered results data: List = [] - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: - data.extend(new_data) + # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling + for input_coro in chunks(input_coroutines, 100): + + results = await asyncio.gather(*input_coro, return_exceptions=True) + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - logger.info("Downloaded data for %s with length %s.", pair, len(data)) + logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, From 30bc96cf3f802a882e4e5e35e0c0df8db876acfc Mon Sep 17 00:00:00 2001 From: sid Date: Sat, 9 Oct 2021 06:36:23 +0530 Subject: [PATCH 32/50] simplify expression --- freqtrade/optimize/hyperopt_loss_max_drawdown.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_max_drawdown.py b/freqtrade/optimize/hyperopt_loss_max_drawdown.py index 6777fb2e8..ce955d928 100644 --- a/freqtrade/optimize/hyperopt_loss_max_drawdown.py +++ b/freqtrade/optimize/hyperopt_loss_max_drawdown.py @@ -38,6 +38,4 @@ class MaxDrawDownHyperOptLoss(IHyperOptLoss): except ValueError: # No losing trade, therefore no drawdown. return -total_profit - max_drawdown_rev = 1 / max_drawdown[0] - ret = max_drawdown_rev * total_profit - return -ret + return -total_profit / max_drawdown[0] From 7b1c888665b214aee599958bf0a5b9656d13531b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Oct 2021 08:39:32 +0200 Subject: [PATCH 33/50] Add FAQ entry for incomplete candles closes #5687 --- docs/faq.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 75c40a681..d9777ddf1 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -54,9 +54,11 @@ you can't say much from few trades. Yes. You can edit your config and use the `/reload_config` command to reload the configuration. The bot will stop, reload the configuration and strategy and will restart with the new configuration and strategy. -### I want to improve the bot with a new strategy +### I want to use incomplete candles -That's great. We have a nice backtesting and hyperoptimization setup. See the tutorial [here|Testing-new-strategies-with-Hyperopt](bot-usage.md#hyperopt-commands). +Freqtrade will not provide incomplete candles to strategies. Using incomplete candles will lead to repainting and consequently to strategies with "ghost" buys, which are impossible to both backtest, and verify after they happened. + +You can use "current" market data by using the [dataprovider](strategy-customization.md#orderbookpair-maximum)'s orderbook or ticker methods - which however cannot be used during backtesting. ### Is there a setting to only SELL the coins being held and not perform anymore BUYS? From 2c68342140979f7ade3e271d6841b9be81590b51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Oct 2021 10:37:33 +0200 Subject: [PATCH 34/50] Move pypi installation to documentation --- README.md | 8 +++++--- docs/installation.md | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 79763983d..0a4d6424e 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Please find the complete documentation on our [website](https://www.freqtrade.io - [x] **Dry-run**: Run the bot without paying money. - [x] **Backtesting**: Run a simulation of your buy/sell strategy. - [x] **Strategy Optimization by machine learning**: Use machine learning to optimize your buy/sell strategy parameters with real exchange data. -- [x] **Edge position sizing** Calculate your win rate, risk reward ratio, the best stoploss and adjust your position size before taking a position for each specific market. [Learn more](https://www.freqtrade.io/en/latest/edge/). +- [x] **Edge position sizing** Calculate your win rate, risk reward ratio, the best stoploss and adjust your position size before taking a position for each specific market. [Learn more](https://www.freqtrade.io/en/stable/edge/). - [x] **Whitelist crypto-currencies**: Select which crypto-currency you want to trade or use dynamic whitelists. - [x] **Blacklist crypto-currencies**: Select which crypto-currency you want to avoid. - [x] **Manageable via Telegram**: Manage the bot with Telegram. @@ -66,10 +66,12 @@ Please find the complete documentation on our [website](https://www.freqtrade.io Freqtrade provides a Linux/macOS script to install all dependencies and help you to configure the bot. ```bash -pip install freqtrade +git clone -b develop https://github.com/freqtrade/freqtrade.git +cd freqtrade +./setup.sh --install ``` -For any other type of installation please refer to [Installation doc](https://www.freqtrade.io/en/latest/installation/). +For any other type of installation please refer to [Installation doc](https://www.freqtrade.io/en/stable/installation/). ## Basic Usage diff --git a/docs/installation.md b/docs/installation.md index 5e4a19d88..d468786d3 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -113,6 +113,13 @@ git checkout develop You may later switch between branches at any time with the `git checkout stable`/`git checkout develop` commands. +??? Note "Install from pypi" + An alternative way to install Freqtrade is from [pypi](https://pypi.org/project/freqtrade/). The downside is that this method requires ta-lib to be correctly installed beforehand, and is therefore currently not the recommended way to install Freqtrade. + + ``` bash + pip install freqtrade + ``` + ------ ## Script Installation From 1a3b41ed9718338c6458de3f87527591337a03cb Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 9 Oct 2021 15:35:39 +0200 Subject: [PATCH 35/50] Rephrase and simplify UI access section in docker quickstart --- docs/docker_quickstart.md | 40 +++++++++------------------------------ 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/docs/docker_quickstart.md b/docs/docker_quickstart.md index cf525b926..95df37811 100644 --- a/docs/docker_quickstart.md +++ b/docs/docker_quickstart.md @@ -70,39 +70,17 @@ docker-compose up -d !!! Warning "Default configuration" While the configuration generated will be mostly functional, you will still need to verify that all options correspond to what you want (like Pricing, pairlist, ...) before starting the bot. -#### Acessing the UI +#### Accessing the UI -Uncommend the 2 lines below and add your IP adress in the following format (like 192.168.2.67:8080:8080) to the ft_userdata/docker-compose.yml: -'''bash - ports: - - "yourIPadress:8080:8080" -''' -Your ft_userdata/user_data/config.json should look like: -'''bash -api_server": { - "enabled": true, - "listen_ip_address": "0.0.0.0", - "listen_port": 8080, - "verbosity": "error", - "enable_openapi": false, - "jwt_secret_key": "****", - "CORS_origins": [], - "username": "****", - "password": "****" - }, -''' -instead of "****" you will have your data in. -Then rebuild your docker file: -Linux: -'''bash -sudo docker-compose down && sudo docker-compose pull && sudo docker-compose build && sudo docker-compose up -d -''' -Windows: -'''bash -docker-compose down && docker-compose pull && docker-compose build && docker-compose up -d -''' +If you've selected to enable FreqUI in the `new-config` step, you will have freqUI available at port `localhost:8080`. -You can now access the UI by typing yourIPadress:8080 in your browser. +You can now access the UI by typing localhost:8080 in your browser. + +??? Note "UI Access on a remote servers" + If you're running on a VPS, you should consider using either a ssh tunnel, or setup a VPN (openVPN, wireguard) to connect to your bot. + This will ensure that freqUI is not directly exposed to the internet, which is not recommended for security reasons (freqUI does not support https out of the box). + Setup of these tools is not part of this tutorial, however many good tutorials can be found on the internet. + Please also read the [API configuration with docker](rest-api.md#configuration-with-docker) section to learn more about this configuration. #### Monitoring the bot From 57095d7167aa481bf184f19f39acfeded8149a7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 03:01:22 +0000 Subject: [PATCH 36/50] Bump wrapt from 1.12.1 to 1.13.1 Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.12.1 to 1.13.1. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.12.1...1.13.1) --- updated-dependencies: - dependency-name: wrapt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fe06b9d2..4eb2bf66b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ arrow==1.1.1 cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 -wrapt==1.12.1 +wrapt==1.13.1 jsonschema==4.0.1 TA-Lib==0.4.21 technical==1.3.0 From 7323ffa25a408a64a94ce577caeece6530bdfa1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 03:01:31 +0000 Subject: [PATCH 37/50] Bump blosc from 1.10.4 to 1.10.6 Bumps [blosc](https://github.com/blosc/python-blosc) from 1.10.4 to 1.10.6. - [Release notes](https://github.com/blosc/python-blosc/releases) - [Changelog](https://github.com/Blosc/python-blosc/blob/master/RELEASE_NOTES.rst) - [Commits](https://github.com/blosc/python-blosc/compare/v1.10.4...v1.10.6) --- updated-dependencies: - dependency-name: blosc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fe06b9d2..3a49977c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ tabulate==0.8.9 pycoingecko==2.2.0 jinja2==3.0.1 tables==3.6.1 -blosc==1.10.4 +blosc==1.10.6 # find first, C search in arrays py_find_1st==1.1.5 From 5fb0401dca6b9ae19c678b0a7577129969901ff9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 03:01:39 +0000 Subject: [PATCH 38/50] Bump cryptography from 3.4.8 to 35.0.0 Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.8 to 35.0.0. - [Release notes](https://github.com/pyca/cryptography/releases) - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/3.4.8...35.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7fe06b9d2..89728f782 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ pandas-ta==0.3.14b ccxt==1.57.38 # Pin cryptography for now due to rust build errors with piwheels -cryptography==3.4.8 +cryptography==35.0.0 aiohttp==3.7.4.post0 SQLAlchemy==1.4.25 python-telegram-bot==13.7 From 3fdc62d29c3f81f4ebdd826e0cd5ad21cde6d1a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:05:14 +0000 Subject: [PATCH 39/50] Bump flake8 from 3.9.2 to 4.0.0 Bumps [flake8](https://github.com/pycqa/flake8) from 3.9.2 to 4.0.0. - [Release notes](https://github.com/pycqa/flake8/releases) - [Commits](https://github.com/pycqa/flake8/compare/3.9.2...4.0.0) --- updated-dependencies: - dependency-name: flake8 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index a3ed37bea..3f2d6035d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,7 +4,7 @@ -r requirements-hyperopt.txt coveralls==3.2.0 -flake8==3.9.2 +flake8==4.0.0 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.4.1 mypy==0.910 From e467491dbecc7f49444c60471485daacf546dfa7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:05:23 +0000 Subject: [PATCH 40/50] Bump jsonschema from 4.0.1 to 4.1.0 Bumps [jsonschema](https://github.com/Julian/jsonschema) from 4.0.1 to 4.1.0. - [Release notes](https://github.com/Julian/jsonschema/releases) - [Changelog](https://github.com/Julian/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/Julian/jsonschema/compare/v4.0.1...v4.1.0) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4eb2bf66b..641264d53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 wrapt==1.13.1 -jsonschema==4.0.1 +jsonschema==4.1.0 TA-Lib==0.4.21 technical==1.3.0 tabulate==0.8.9 From afc086f33c85162107a27dda013544d37ced3a51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:05:30 +0000 Subject: [PATCH 41/50] Bump arrow from 1.1.1 to 1.2.0 Bumps [arrow](https://github.com/arrow-py/arrow) from 1.1.1 to 1.2.0. - [Release notes](https://github.com/arrow-py/arrow/releases) - [Changelog](https://github.com/arrow-py/arrow/blob/master/CHANGELOG.rst) - [Commits](https://github.com/arrow-py/arrow/commits) --- updated-dependencies: - dependency-name: arrow dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4eb2bf66b..90ef02ae5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ cryptography==3.4.8 aiohttp==3.7.4.post0 SQLAlchemy==1.4.25 python-telegram-bot==13.7 -arrow==1.1.1 +arrow==1.2.0 cachetools==4.2.2 requests==2.26.0 urllib3==1.26.7 From 32174f8f906304f61e7f21afa3669eab4458dc5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:14:08 +0000 Subject: [PATCH 42/50] Bump pyjwt from 2.1.0 to 2.2.0 Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.1.0 to 2.2.0. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.1.0...2.2.0) --- updated-dependencies: - dependency-name: pyjwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4eb2bf66b..133aee144 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ tabulate==0.8.9 pycoingecko==2.2.0 jinja2==3.0.1 tables==3.6.1 -blosc==1.10.4 +blosc==1.10.6 # find first, C search in arrays py_find_1st==1.1.5 @@ -34,7 +34,7 @@ sdnotify==0.3.2 # API Server fastapi==0.68.1 uvicorn==0.15.0 -pyjwt==2.1.0 +pyjwt==2.2.0 aiofiles==0.7.0 psutil==5.8.0 From 4921a4caecd0cfbffa7df3894f4649876d4dd28a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:14:19 +0000 Subject: [PATCH 43/50] Bump jinja2 from 3.0.1 to 3.0.2 Bumps [jinja2](https://github.com/pallets/jinja) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.0.1...3.0.2) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1d9aaf4b5..e2de4f629 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ TA-Lib==0.4.21 technical==1.3.0 tabulate==0.8.9 pycoingecko==2.2.0 -jinja2==3.0.1 +jinja2==3.0.2 tables==3.6.1 blosc==1.10.6 From 90ea3d444060359320ec534044785699617e0cb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 05:18:24 +0000 Subject: [PATCH 44/50] Bump mkdocs-material from 7.3.1 to 7.3.2 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.3.1 to 7.3.2. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.3.1...7.3.2) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index bbbb240ba..9a733d8f7 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.3.1 +mkdocs-material==7.3.2 mdx_truly_sane_lists==1.2 pymdown-extensions==9.0 From 29371b2f28591a611e039bad4da718ae726dfb23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 06:15:03 +0000 Subject: [PATCH 45/50] Bump joblib from 1.0.1 to 1.1.0 Bumps [joblib](https://github.com/joblib/joblib) from 1.0.1 to 1.1.0. - [Release notes](https://github.com/joblib/joblib/releases) - [Changelog](https://github.com/joblib/joblib/blob/master/CHANGES.rst) - [Commits](https://github.com/joblib/joblib/compare/1.0.1...1.1.0) --- updated-dependencies: - dependency-name: joblib dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index b4067d1db..96690bcbb 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -6,6 +6,6 @@ scipy==1.7.1 scikit-learn==0.24.2 scikit-optimize==0.8.1 filelock==3.3.0 -joblib==1.0.1 +joblib==1.1.0 psutil==5.8.0 progressbar2==3.53.3 From 802599bdc99cdd82e390d51cdb000c87d03e96a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 06:15:32 +0000 Subject: [PATCH 46/50] Bump ccxt from 1.57.38 to 1.57.94 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.57.38 to 1.57.94. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.57.38...1.57.94) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 788801987..ff9ea9423 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.21.2 pandas==1.3.3 pandas-ta==0.3.14b -ccxt==1.57.38 +ccxt==1.57.94 # Pin cryptography for now due to rust build errors with piwheels cryptography==35.0.0 aiohttp==3.7.4.post0 @@ -34,7 +34,7 @@ sdnotify==0.3.2 # API Server fastapi==0.68.1 uvicorn==0.15.0 -pyjwt==2.1.0 +pyjwt==2.2.0 aiofiles==0.7.0 psutil==5.8.0 From fa00b52c4742cbe7b2ec4c583ee5a5828ca00ff7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Oct 2021 07:41:16 +0000 Subject: [PATCH 47/50] Bump scikit-learn from 0.24.2 to 1.0 Bumps [scikit-learn](https://github.com/scikit-learn/scikit-learn) from 0.24.2 to 1.0. - [Release notes](https://github.com/scikit-learn/scikit-learn/releases) - [Commits](https://github.com/scikit-learn/scikit-learn/compare/0.24.2...1.0) --- updated-dependencies: - dependency-name: scikit-learn dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index 96690bcbb..edd078e9e 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -3,7 +3,7 @@ # Required for hyperopt scipy==1.7.1 -scikit-learn==0.24.2 +scikit-learn==1.0 scikit-optimize==0.8.1 filelock==3.3.0 joblib==1.1.0 From 396bc9b2e3d33993052a3b2038cad9ee19d3736d Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 11 Oct 2021 20:00:53 +0200 Subject: [PATCH 48/50] Version bump flake8-tidy-imports to 4.5.0 --- requirements-dev.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 3f2d6035d..74ebee479 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,8 +5,7 @@ coveralls==3.2.0 flake8==4.0.0 -flake8-type-annotations==0.1.0 -flake8-tidy-imports==4.4.1 +flake8-tidy-imports==4.5.0 mypy==0.910 pytest==6.2.5 pytest-asyncio==0.15.1 From ce9debe9fd89bab9171f35d3632f88f27c0d080a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Oct 2021 06:44:07 +0200 Subject: [PATCH 49/50] Add version argument to freqUI installer --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 6 ++++++ freqtrade/commands/deploy_commands.py | 16 ++++++++++++---- tests/commands/test_commands.py | 27 ++++++++++++++++++++++----- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 9643705a5..a02faa736 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -73,7 +73,7 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", "trade_source", "timeframe", "plot_auto_open"] -ARGS_INSTALL_UI = ["erase_ui_only"] +ARGS_INSTALL_UI = ["erase_ui_only", 'ui_version'] ARGS_SHOW_TRADES = ["db_url", "trade_ids", "print_json"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index d350a9426..30a9b0137 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -414,6 +414,12 @@ AVAILABLE_CLI_OPTIONS = { action='store_true', default=False, ), + "ui_version": Arg( + '--ui-version', + help=('Specify a specific version of FreqUI to install. ' + 'Not specifying this installs the latest version.'), + type=str, + ), # Templating options "template": Arg( '--template', diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 4f9e5bbad..92c9adf66 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -128,7 +128,7 @@ def download_and_install_ui(dest_folder: Path, dl_url: str, version: str): f.write(version) -def get_ui_download_url() -> Tuple[str, str]: +def get_ui_download_url(version: Optional[str] = None) -> Tuple[str, str]: base_url = 'https://api.github.com/repos/freqtrade/frequi/' # Get base UI Repo path @@ -136,8 +136,16 @@ def get_ui_download_url() -> Tuple[str, str]: resp.raise_for_status() r = resp.json() - latest_version = r[0]['name'] - assets = r[0].get('assets', []) + if version: + tmp = [x for x in r if x['name'] == version] + if tmp: + latest_version = tmp[0]['name'] + assets = tmp[0].get('assets', []) + else: + raise ValueError("UI-Version not found.") + else: + latest_version = r[0]['name'] + assets = r[0].get('assets', []) dl_url = '' if assets and len(assets) > 0: dl_url = assets[0]['browser_download_url'] @@ -156,7 +164,7 @@ def start_install_ui(args: Dict[str, Any]) -> None: dest_folder = Path(__file__).parents[1] / 'rpc/api_server/ui/installed/' # First make sure the assets are removed. - dl_url, latest_version = get_ui_download_url() + dl_url, latest_version = get_ui_download_url(args.get('ui_version')) curr_version = read_ui_version(dest_folder) if curr_version == latest_version and not args.get('erase_ui_only'): diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 8889617ba..6a0e741d9 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -605,16 +605,33 @@ def test_get_ui_download_url(mocker): def test_get_ui_download_url_direct(mocker): response = MagicMock() response.json = MagicMock( - side_effect=[[{ - 'assets_url': 'http://whatever.json', - 'name': '0.0.1', - 'assets': [{'browser_download_url': 'http://download11.zip'}]}]]) + return_value=[ + { + 'assets_url': 'http://whatever.json', + 'name': '0.0.2', + 'assets': [{'browser_download_url': 'http://download22.zip'}] + }, + { + 'assets_url': 'http://whatever.json', + 'name': '0.0.1', + 'assets': [{'browser_download_url': 'http://download1.zip'}] + }, + ]) get_mock = mocker.patch("freqtrade.commands.deploy_commands.requests.get", return_value=response) x, last_version = get_ui_download_url() assert get_mock.call_count == 1 + assert last_version == '0.0.2' + assert x == 'http://download22.zip' + get_mock.reset_mock() + response.json.reset_mock() + + x, last_version = get_ui_download_url('0.0.1') assert last_version == '0.0.1' - assert x == 'http://download11.zip' + assert x == 'http://download1.zip' + + with pytest.raises(ValueError, match="UI-Version not found."): + x, last_version = get_ui_download_url('0.0.3') def test_download_data_keyboardInterrupt(mocker, caplog, markets): From 8798ae567744ceefed92577acbd66fb974df6b15 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 12 Oct 2021 19:06:23 +0200 Subject: [PATCH 50/50] Version bump also scikit-optimize --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index edd078e9e..e97e78638 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -4,7 +4,7 @@ # Required for hyperopt scipy==1.7.1 scikit-learn==1.0 -scikit-optimize==0.8.1 +scikit-optimize==0.9.0 filelock==3.3.0 joblib==1.1.0 psutil==5.8.0