From 93c1dff71bbae773b922794f956013e0d0b9c908 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Sep 2021 20:43:07 +0200 Subject: [PATCH 01/58] Allow adding new additional headers --- freqtrade/exchange/exchange.py | 9 ++++++++- tests/exchange/test_exchange.py | 7 +++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ecf3302d8..3dc295563 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -54,6 +54,9 @@ class Exchange: # Parameters to add directly to buy/sell calls (like agreeing to trading agreement) _params: Dict = {} + # Additional headers - added to the ccxt object + _headers: Dict = {} + # Dict to specify which options each exchange implements # This defines defaults, which can be selectively overridden by subclasses using _ft_has # or by specifying them in the configuration. @@ -169,7 +172,7 @@ class Exchange: asyncio.get_event_loop().run_until_complete(self._api_async.close()) def _init_ccxt(self, exchange_config: Dict[str, Any], ccxt_module: CcxtModuleType = ccxt, - ccxt_kwargs: dict = None) -> ccxt.Exchange: + ccxt_kwargs: Dict = {}) -> ccxt.Exchange: """ Initialize ccxt with given config and return valid ccxt instance. @@ -188,6 +191,10 @@ class Exchange: } if ccxt_kwargs: logger.info('Applying additional ccxt config: %s', ccxt_kwargs) + if self._headers: + # Inject static headers after the above output to not confuse users. + ccxt_kwargs = deep_merge_dicts({'headers': self._headers}, ccxt_kwargs) + if ccxt_kwargs: ex_config.update(ccxt_kwargs) try: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 42da5dddc..144063c07 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -108,6 +108,13 @@ def test_init_ccxt_kwargs(default_conf, mocker, caplog): assert hasattr(ex._api_async, 'TestKWARG') assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) assert log_has(asynclogmsg, caplog) + # Test additional headers case + Exchange._headers = {'hello': 'world'} + ex = Exchange(conf) + + assert log_has("Applying additional ccxt config: {'TestKWARG': 11, 'TestKWARG44': 11}", caplog) + assert ex._api.headers == {'hello': 'world'} + Exchange._headers = {} def test_destroy(default_conf, mocker, caplog): From 19ad1654836992365af7536189095662205d768a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 08:48:53 +0200 Subject: [PATCH 02/58] Add time_in_force for kucoin --- docs/configuration.md | 4 ++-- docs/exchanges.md | 5 +++++ freqtrade/exchange/binance.py | 1 + freqtrade/exchange/exchange.py | 4 +++- freqtrade/exchange/kucoin.py | 2 ++ 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 09198e019..6ccea4c73 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -444,8 +444,8 @@ The possible values are: `gtc` (default), `fok` or `ioc`. ``` !!! Warning - This is ongoing work. For now, it is supported only for binance. - Please don't change the default value unless you know what you are doing and have researched the impact of using different values. + This is ongoing work. For now, it is supported only for binance and kucoin. + Please don't change the default value unless you know what you are doing and have researched the impact of using different values for your particular exchange. ### Exchange configuration diff --git a/docs/exchanges.md b/docs/exchanges.md index 5f54a524e..81ad670a3 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -4,6 +4,9 @@ This page combines common gotchas and informations which are exchange-specific a ## Binance +Binance supports [time_in_force](configuration.md#understand-order_time_in_force). + + !!! Tip "Stoploss on Exchange" Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. @@ -115,6 +118,8 @@ Kucoin requires a passphrase for each api key, you will therefore need to add th "password": "your_exchange_api_key_password", ``` +Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force). + ### Kucoin Blacklists For Kucoin, please add `"KCS/"` to your blacklist to avoid issues. diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 0c470cb24..189f5f481 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -18,6 +18,7 @@ class Binance(Exchange): _ft_has: Dict = { "stoploss_on_exchange": True, "order_time_in_force": ['gtc', 'fok', 'ioc'], + "time_in_force_parameter": "timeInForce", "ohlcv_candle_limit": 1000, "trades_pagination": "id", "trades_pagination_arg": "fromId", diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 3dc295563..80f20b17e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -63,6 +63,7 @@ class Exchange: _ft_has_default: Dict = { "stoploss_on_exchange": False, "order_time_in_force": ["gtc"], + "time_in_force_parameter": "timeInForce", "ohlcv_params": {}, "ohlcv_candle_limit": 500, "ohlcv_partial_candle": True, @@ -723,7 +724,8 @@ class Exchange: params = self._params.copy() if time_in_force != 'gtc' and ordertype != 'market': - params.update({'timeInForce': time_in_force}) + param = self._ft_has.get('time_in_force_parameter', '') + params.update({param: time_in_force}) try: # Set the precision for amount and price(rate) as accepted by the exchange diff --git a/freqtrade/exchange/kucoin.py b/freqtrade/exchange/kucoin.py index 22886a1d8..5d818f6a2 100644 --- a/freqtrade/exchange/kucoin.py +++ b/freqtrade/exchange/kucoin.py @@ -21,4 +21,6 @@ class Kucoin(Exchange): _ft_has: Dict = { "l2_limit_range": [20, 100], "l2_limit_range_required": False, + "order_time_in_force": ['gtc', 'fok', 'ioc'], + "time_in_force_parameter": "timeInForce", } From e64ccd8fc111e6d5eeeb193509317f0be00f1d80 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 08:54:15 +0200 Subject: [PATCH 03/58] Add new_config section for kucoin --- freqtrade/commands/build_config_commands.py | 8 ++++++++ .../templates/subtemplates/exchange_kucoin.j2 | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 freqtrade/templates/subtemplates/exchange_kucoin.j2 diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 1fe90e83a..780ad4161 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -105,6 +105,8 @@ def ask_user_config() -> Dict[str, Any]: "bittrex", "kraken", "ftx", + "kucoin", + "gateio", Separator(), "other", ], @@ -128,6 +130,12 @@ def ask_user_config() -> Dict[str, Any]: "message": "Insert Exchange Secret", "when": lambda x: not x['dry_run'] }, + { + "type": "password", + "name": "exchange_key_password", + "message": "Insert Exchange API Key password", + "when": lambda x: not x['dry_run'] and x['exchange_name'] == 'kucoin' + }, { "type": "confirm", "name": "telegram", diff --git a/freqtrade/templates/subtemplates/exchange_kucoin.j2 b/freqtrade/templates/subtemplates/exchange_kucoin.j2 new file mode 100644 index 000000000..f9dfff663 --- /dev/null +++ b/freqtrade/templates/subtemplates/exchange_kucoin.j2 @@ -0,0 +1,18 @@ +"exchange": { + "name": "{{ exchange_name | lower }}", + "key": "{{ exchange_key }}", + "secret": "{{ exchange_secret }}", + "password": "{{ exchange_key_password }}", + "ccxt_config": { + "enableRateLimit": true + "rateLimit": 200 + }, + "ccxt_async_config": { + "enableRateLimit": true, + "rateLimit": 200 + }, + "pair_whitelist": [ + ], + "pair_blacklist": [ + ] +} From 68f13173bcaf4d88c2835abc66f9d0b92bf7ce7e Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 08:59:08 +0200 Subject: [PATCH 04/58] Update new-config templates to use USDT by default --- freqtrade/commands/build_config_commands.py | 4 ++-- .../subtemplates/exchange_binance.j2 | 15 ------------- .../subtemplates/exchange_bittrex.j2 | 10 --------- .../templates/subtemplates/exchange_kraken.j2 | 22 ++----------------- 4 files changed, 4 insertions(+), 47 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 780ad4161..faa8a98f4 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -61,13 +61,13 @@ def ask_user_config() -> Dict[str, Any]: "type": "text", "name": "stake_currency", "message": "Please insert your stake currency:", - "default": 'BTC', + "default": 'USDT', }, { "type": "text", "name": "stake_amount", "message": f"Please insert your stake amount (Number or '{UNLIMITED_STAKE_AMOUNT}'):", - "default": "0.01", + "default": "100", "validate": lambda val: val == UNLIMITED_STAKE_AMOUNT or validate_is_float(val), "filter": lambda val: '"' + UNLIMITED_STAKE_AMOUNT + '"' if val == UNLIMITED_STAKE_AMOUNT diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index 38ba4fa5c..217002a7c 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -8,21 +8,6 @@ "rateLimit": 200 }, "pair_whitelist": [ - "ALGO/BTC", - "ATOM/BTC", - "BAT/BTC", - "BCH/BTC", - "BRD/BTC", - "EOS/BTC", - "ETH/BTC", - "IOTA/BTC", - "LINK/BTC", - "LTC/BTC", - "NEO/BTC", - "NXS/BTC", - "XMR/BTC", - "XRP/BTC", - "XTZ/BTC" ], "pair_blacklist": [ "BNB/BTC", diff --git a/freqtrade/templates/subtemplates/exchange_bittrex.j2 b/freqtrade/templates/subtemplates/exchange_bittrex.j2 index 7b27318ca..0394790ce 100644 --- a/freqtrade/templates/subtemplates/exchange_bittrex.j2 +++ b/freqtrade/templates/subtemplates/exchange_bittrex.j2 @@ -15,16 +15,6 @@ "rateLimit": 500 }, "pair_whitelist": [ - "ETH/BTC", - "LTC/BTC", - "ETC/BTC", - "DASH/BTC", - "ZEC/BTC", - "XLM/BTC", - "XRP/BTC", - "TRX/BTC", - "ADA/BTC", - "XMR/BTC" ], "pair_blacklist": [ ] diff --git a/freqtrade/templates/subtemplates/exchange_kraken.j2 b/freqtrade/templates/subtemplates/exchange_kraken.j2 index 7139a0830..4d0e4c1ff 100644 --- a/freqtrade/templates/subtemplates/exchange_kraken.j2 +++ b/freqtrade/templates/subtemplates/exchange_kraken.j2 @@ -7,28 +7,10 @@ "ccxt_async_config": { "enableRateLimit": true, "rateLimit": 1000 + // Enable the below for downoading data. + //"rateLimit": 3100 }, "pair_whitelist": [ - "ADA/EUR", - "ATOM/EUR", - "BAT/EUR", - "BCH/EUR", - "BTC/EUR", - "DAI/EUR", - "DASH/EUR", - "EOS/EUR", - "ETC/EUR", - "ETH/EUR", - "LINK/EUR", - "LTC/EUR", - "QTUM/EUR", - "REP/EUR", - "WAVES/EUR", - "XLM/EUR", - "XMR/EUR", - "XRP/EUR", - "XTZ/EUR", - "ZEC/EUR" ], "pair_blacklist": [ From c489e6825c3cc0affeb46e2a2f1f78d119e933f5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 09:18:15 +0200 Subject: [PATCH 05/58] Simplify binance blacklist --- .../templates/subtemplates/exchange_binance.j2 | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index 217002a7c..3022464c7 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -10,17 +10,6 @@ "pair_whitelist": [ ], "pair_blacklist": [ - "BNB/BTC", - "BNB/BUSD", - "BNB/ETH", - "BNB/EUR", - "BNB/NGN", - "BNB/PAX", - "BNB/RUB", - "BNB/TRY", - "BNB/TUSD", - "BNB/USDC", - "BNB/USDS", - "BNB/USDT" + "BNB/.*", ] } From b4130dfabbdd0a68119080f33c990b34d4116d77 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 09:23:26 +0200 Subject: [PATCH 06/58] Use volumePairlist instead of staticPairlist in generated config --- freqtrade/templates/base_config.json.j2 | 9 ++++++++- freqtrade/templates/subtemplates/exchange_binance.j2 | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/templates/base_config.json.j2 b/freqtrade/templates/base_config.json.j2 index a5782f7cd..68eebdbd4 100644 --- a/freqtrade/templates/base_config.json.j2 +++ b/freqtrade/templates/base_config.json.j2 @@ -1,3 +1,10 @@ +{%set volume_pairlist = '{ + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "min_value": 0, + "refresh_period": 1800 + }' %} { "max_open_trades": {{ max_open_trades }}, "stake_currency": "{{ stake_currency }}", @@ -29,7 +36,7 @@ }, {{ exchange | indent(4) }}, "pairlists": [ - {"method": "StaticPairList"} + {{ '{"method": "StaticPairList"}' if exchange_name == 'bittrex' else volume_pairlist }} ], "edge": { "enabled": false, diff --git a/freqtrade/templates/subtemplates/exchange_binance.j2 b/freqtrade/templates/subtemplates/exchange_binance.j2 index 3022464c7..de58b6f72 100644 --- a/freqtrade/templates/subtemplates/exchange_binance.j2 +++ b/freqtrade/templates/subtemplates/exchange_binance.j2 @@ -10,6 +10,6 @@ "pair_whitelist": [ ], "pair_blacklist": [ - "BNB/.*", + "BNB/.*" ] } From 2f92838c39639e4770b95258ca0357d4e69c7d40 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 16:18:47 +0200 Subject: [PATCH 07/58] Properly close parenteses in exchange doc --- docs/exchanges.md | 5 ++++- mkdocs.yml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 81ad670a3..42a850acd 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -6,7 +6,6 @@ This page combines common gotchas and informations which are exchange-specific a Binance supports [time_in_force](configuration.md#understand-order_time_in_force). - !!! Tip "Stoploss on Exchange" Binance supports `stoploss_on_exchange` and uses stop-loss-limit orders. It provides great advantages, so we recommend to benefit from it. @@ -116,6 +115,8 @@ Kucoin requires a passphrase for each api key, you will therefore need to add th "key": "your_exchange_key", "secret": "your_exchange_secret", "password": "your_exchange_api_key_password", + // ... +} ``` Kucoin supports [time_in_force](configuration.md#understand-order_time_in_force). @@ -163,6 +164,8 @@ For example, to test the order type `FOK` with Kraken, and modify candle limit t "order_time_in_force": ["gtc", "fok"], "ohlcv_candle_limit": 200 } + //... +} ``` !!! Warning diff --git a/mkdocs.yml b/mkdocs.yml index 854939ca0..05156168f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,10 +23,10 @@ nav: - Hyperopt: hyperopt.md - Utility Sub-commands: utils.md - Plotting: plotting.md + - Exchange-specific Notes: exchanges.md - Data Analysis: - Jupyter Notebooks: data-analysis.md - Strategy analysis: strategy_analysis_example.md - - Exchange-specific Notes: exchanges.md - Advanced Topics: - Advanced Post-installation Tasks: advanced-setup.md - Edge Positioning: edge.md From 103a8e827ea7e280b93d806d355ca1157ed55692 Mon Sep 17 00:00:00 2001 From: Rikj000 Date: Fri, 3 Sep 2021 16:37:36 +0200 Subject: [PATCH 08/58] =?UTF-8?q?=E2=9A=A1=20`setup.sh`=20-=20Use=20`build?= =?UTF-8?q?=5Fhelpers/install=5Fta-lib.sh`=20for=20TA-Lib=20installation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build_helpers/install_ta-lib.sh | 2 +- setup.sh | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index dd87cf105..590b7d961 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -13,7 +13,7 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then && ./configure --prefix=${INSTALL_LOC}/ \ && make -j$(nproc) \ && which sudo && sudo make install || make install \ - && cd .. + && cd .. && rm -rf ./ta-lib/ else echo "TA-lib already installed, skipping installation" fi diff --git a/setup.sh b/setup.sh index e5f81578d..217500569 100755 --- a/setup.sh +++ b/setup.sh @@ -95,19 +95,7 @@ function install_talib() { return fi - cd build_helpers - tar zxvf ta-lib-0.4.0-src.tar.gz - cd ta-lib - sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h - ./configure --prefix=/usr/local - make - sudo make install - if [ -x "$(command -v apt-get)" ]; then - echo "Updating library path using ldconfig" - sudo ldconfig - fi - cd .. && rm -rf ./ta-lib/ - cd .. + cd build_helpers && ./install_ta-lib.sh && cd .. } function install_mac_newer_python_dependencies() { From 493fb3507316296f90f6c6384a1e84d366b82efa Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Sep 2021 22:05:40 +0200 Subject: [PATCH 09/58] Fix uvicorn not working properly on windows --- freqtrade/rpc/api_server/uvicorn_threaded.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/freqtrade/rpc/api_server/uvicorn_threaded.py b/freqtrade/rpc/api_server/uvicorn_threaded.py index b63999f51..79af659c7 100644 --- a/freqtrade/rpc/api_server/uvicorn_threaded.py +++ b/freqtrade/rpc/api_server/uvicorn_threaded.py @@ -5,6 +5,20 @@ import time import uvicorn +def asyncio_setup() -> None: # pragma: no cover + # Set eventloop for win32 setups + # Reverts a change done in uvicorn 0.15.0 - which now sets the eventloop + # via policy. + import sys + + if sys.version_info >= (3, 8) and sys.platform == "win32": + import asyncio + import selectors + selector = selectors.SelectSelector() + loop = asyncio.SelectorEventLoop(selector) + asyncio.set_event_loop(loop) + + class UvicornServer(uvicorn.Server): """ Multithreaded server - as found in https://github.com/encode/uvicorn/issues/742 @@ -28,7 +42,7 @@ class UvicornServer(uvicorn.Server): try: import uvloop # noqa except ImportError: # pragma: no cover - from uvicorn.loops.asyncio import asyncio_setup + asyncio_setup() else: asyncio.set_event_loop(uvloop.new_event_loop()) From eb0362c29ef7f125db4fda5590485c4c9f002f7b Mon Sep 17 00:00:00 2001 From: Rik Helsen Date: Fri, 3 Sep 2021 23:52:40 +0200 Subject: [PATCH 10/58] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20=EF=B8=8F`install=5F?= =?UTF-8?q?ta-lib.sh`=20-=20Run=20`ldconfig`=20after=20`make=20install`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build_helpers/install_ta-lib.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build_helpers/install_ta-lib.sh b/build_helpers/install_ta-lib.sh index 590b7d961..d12b16364 100755 --- a/build_helpers/install_ta-lib.sh +++ b/build_helpers/install_ta-lib.sh @@ -12,9 +12,12 @@ if [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \ && ./configure --prefix=${INSTALL_LOC}/ \ && make -j$(nproc) \ - && which sudo && sudo make install || make install \ - && cd .. && rm -rf ./ta-lib/ + && which sudo && sudo make install || make install + if [ -x "$(command -v apt-get)" ]; then + echo "Updating library path using ldconfig" + sudo ldconfig + fi + cd .. && rm -rf ./ta-lib/ else echo "TA-lib already installed, skipping installation" fi -# && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \ From 4e049f65f227c7e2bf103db5cd9f97988c7a2250 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Sep 2021 09:14:44 +0200 Subject: [PATCH 11/58] Exclude some parts from coverage that can't really be tested --- freqtrade/__init__.py | 2 +- freqtrade/loggers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index e96e7f530..2747efc96 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -22,7 +22,7 @@ if __version__ == 'develop': # subprocess.check_output( # ['git', 'log', '--format="%h"', '-n 1'], # stderr=subprocess.DEVNULL).decode("utf-8").rstrip().strip('"') - except Exception: + except Exception: # pragma: no cover # git not available, ignore try: # Try Fallback to freqtrade_commit file (created by CI while building docker image) diff --git a/freqtrade/loggers.py b/freqtrade/loggers.py index fbb05d879..5c5831695 100644 --- a/freqtrade/loggers.py +++ b/freqtrade/loggers.py @@ -87,7 +87,7 @@ def setup_logging(config: Dict[str, Any]) -> None: # syslog config. The messages should be equal for this. handler_sl.setFormatter(Formatter('%(name)s - %(levelname)s - %(message)s')) logging.root.addHandler(handler_sl) - elif s[0] == 'journald': + elif s[0] == 'journald': # pragma: no cover try: from systemd.journal import JournaldLogHandler except ImportError: From 2173ff0133db834cbda72a2d35fa6a07e5d574b9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Sep 2021 09:16:19 +0200 Subject: [PATCH 12/58] Update PR template to not link to issues in changelog --- .github/PULL_REQUEST_TEMPLATE.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 20ef27f0f..7c0655b20 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,14 +2,16 @@ Thank you for sending your pull request. But first, have you included unit tests, and is your code PEP8 conformant? [More details](https://github.com/freqtrade/freqtrade/blob/develop/CONTRIBUTING.md) ## Summary + Explain in one sentence the goal of this PR Solve the issue: #___ ## Quick changelog -- -- +- +- ## What's new? + *Explain in details what this PR solve or improve. You can include visuals.* From a8f28ffb11a48ce1a8bbb26b209bfd9fa34705d0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Sep 2021 15:34:57 +0200 Subject: [PATCH 13/58] Increase test coverage --- tests/strategy/test_interface.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 128599668..250dcf63d 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -735,11 +735,16 @@ def test_auto_hyperopt_interface(default_conf): PairLocks.timeframe = default_conf['timeframe'] strategy = StrategyResolver.load_strategy(default_conf) + with pytest.raises(OperationalException): + next(strategy.enumerate_parameters('deadBeef')) + assert strategy.buy_rsi.value == strategy.buy_params['buy_rsi'] # PlusDI is NOT in the buy-params, so default should be used assert strategy.buy_plusdi.value == 0.5 assert strategy.sell_rsi.value == strategy.sell_params['sell_rsi'] + assert repr(strategy.sell_rsi) == 'IntParameter(74)' + # Parameter is disabled - so value from sell_param dict will NOT be used. assert strategy.sell_minusdi.value == 0.5 all_params = strategy.detect_all_parameters() From c519ecf8df2fae0e8d789c5abacb6662620d73e7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Sep 2021 15:40:21 +0200 Subject: [PATCH 14/58] Exclude more untestable sections from coverage --- freqtrade/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/main.py b/freqtrade/main.py index 2fd3d32bb..6593fbcb6 100755 --- a/freqtrade/main.py +++ b/freqtrade/main.py @@ -9,7 +9,7 @@ from typing import Any, List # check min. python version -if sys.version_info < (3, 7): +if sys.version_info < (3, 7): # pragma: no cover sys.exit("Freqtrade requires Python version >= 3.7") from freqtrade.commands import Arguments @@ -46,7 +46,7 @@ def main(sysargv: List[str] = None) -> None: "`freqtrade --help` or `freqtrade --help`." ) - except SystemExit as e: + except SystemExit as e: # pragma: no cover return_code = e except KeyboardInterrupt: logger.info('SIGINT received, aborting ...') @@ -60,5 +60,5 @@ def main(sysargv: List[str] = None) -> None: sys.exit(return_code) -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover main() From 1d24d3d5ee68624f2706e2a45ea939c87fa0b3ab Mon Sep 17 00:00:00 2001 From: lenik terenin Date: Sun, 5 Sep 2021 22:41:58 +0900 Subject: [PATCH 15/58] case insensitive blacklist Allow "btc/usdt" pairs in blacklist to match to "BTC/USDT" pairs that come from the exchange. --- freqtrade/plugins/pairlist/pairlist_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/plugins/pairlist/pairlist_helpers.py b/freqtrade/plugins/pairlist/pairlist_helpers.py index 924bfb293..1de27fcbd 100644 --- a/freqtrade/plugins/pairlist/pairlist_helpers.py +++ b/freqtrade/plugins/pairlist/pairlist_helpers.py @@ -17,7 +17,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], if keep_invalid: for pair_wc in wildcardpl: try: - comp = re.compile(pair_wc) + comp = re.compile(pair_wc, re.IGNORECASE) result_partial = [ pair for pair in available_pairs if re.fullmatch(comp, pair) ] @@ -33,7 +33,7 @@ def expand_pairlist(wildcardpl: List[str], available_pairs: List[str], else: for pair_wc in wildcardpl: try: - comp = re.compile(pair_wc) + comp = re.compile(pair_wc, re.IGNORECASE) result += [ pair for pair in available_pairs if re.fullmatch(comp, pair) ] From 4daa4b9e63960da073e8e2d5f7e00906d4117cf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 03:01:10 +0000 Subject: [PATCH 16/58] Bump pytest from 6.2.4 to 6.2.5 Bumps [pytest](https://github.com/pytest-dev/pytest) from 6.2.4 to 6.2.5. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/6.2.4...6.2.5) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 67ee0035b..34d5607f3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ flake8==3.9.2 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.4.1 mypy==0.910 -pytest==6.2.4 +pytest==6.2.5 pytest-asyncio==0.15.1 pytest-cov==2.12.1 pytest-mock==3.6.1 From 771193cbe47b0bd812e87662bdc141b911e316a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 03:01:19 +0000 Subject: [PATCH 17/58] Bump plotly from 5.3.0 to 5.3.1 Bumps [plotly](https://github.com/plotly/plotly.py) from 5.3.0 to 5.3.1. - [Release notes](https://github.com/plotly/plotly.py/releases) - [Changelog](https://github.com/plotly/plotly.py/blob/master/CHANGELOG.md) - [Commits](https://github.com/plotly/plotly.py/compare/v5.3.0...v5.3.1) --- updated-dependencies: - dependency-name: plotly dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-plot.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-plot.txt b/requirements-plot.txt index 62836a729..8e17232b0 100644 --- a/requirements-plot.txt +++ b/requirements-plot.txt @@ -1,5 +1,5 @@ # Include all requirements to run the bot. -r requirements.txt -plotly==5.3.0 +plotly==5.3.1 From 44f8d7abf2ba21c3eb0e1c0f5804d8bafb9d2e31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 03:01:23 +0000 Subject: [PATCH 18/58] Bump ccxt from 1.55.56 to 1.55.83 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.55.56 to 1.55.83. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.55.56...1.55.83) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f77edddfe..e2bed0f9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.21.2 pandas==1.3.2 -ccxt==1.55.56 +ccxt==1.55.83 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 From 765e72715bf739504e53140cd29eb24ef40dda44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Sep 2021 03:01:28 +0000 Subject: [PATCH 19/58] Bump mkdocs-material from 7.2.5 to 7.2.6 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.2.5 to 7.2.6. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.2.5...7.2.6) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index d820c9412..9927740c2 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2.2 -mkdocs-material==7.2.5 +mkdocs-material==7.2.6 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From a04875eb55beab2f9d86534434daf2465b9f0d78 Mon Sep 17 00:00:00 2001 From: EnzovdWetering <36194995+EnzovdWetering@users.noreply.github.com> Date: Mon, 6 Sep 2021 17:53:44 +0200 Subject: [PATCH 20/58] Update edge.md Typo fix --- docs/edge.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/edge.md b/docs/edge.md index 237ff36f6..4402d767f 100644 --- a/docs/edge.md +++ b/docs/edge.md @@ -3,7 +3,7 @@ The `Edge Positioning` module uses probability to calculate your win rate and risk reward ratio. It will use these statistics to control your strategy trade entry points, position size and, stoploss. !!! Warning - WHen using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data. + When using `Edge positioning` with a dynamic whitelist (VolumePairList), make sure to also use `AgeFilter` and set it to at least `calculate_since_number_of_days` to avoid problems with missing data. !!! Note `Edge Positioning` only considers *its own* buy/sell/stoploss signals. It ignores the stoploss, trailing stoploss, and ROI settings in the strategy configuration file. From 6bd495a32a58b6da236f53da2d166f683a21cfff Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Sep 2021 19:53:32 +0200 Subject: [PATCH 21/58] Fix 0Exception error happens when wrong stake-currency is selected and /profit is called --- freqtrade/rpc/rpc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 95a37452b..ca2e84e48 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -403,8 +403,11 @@ class RPC: # Doing the sum is not right - overall profit needs to be based on initial capital profit_all_ratio_sum = sum(profit_all_ratio) if profit_all_ratio else 0.0 starting_balance = self._freqtrade.wallets.get_starting_balance() - profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance - profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance + profit_closed_ratio_fromstart = 0 + profit_all_ratio_fromstart = 0 + if starting_balance: + profit_closed_ratio_fromstart = profit_closed_coin_sum / starting_balance + profit_all_ratio_fromstart = profit_all_coin_sum / starting_balance profit_all_fiat = self._fiat_converter.convert_amount( profit_all_coin_sum, From 10d0987f49b8179f840895882bccebb732098192 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Sep 2021 19:54:34 +0200 Subject: [PATCH 22/58] Fix docs for custom hyperopt space --- docs/advanced-hyperopt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 8f233438b..4a2bafd2e 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -80,7 +80,7 @@ To override a pre-defined space (`roi_space`, `generate_roi_table`, `stoploss_sp class MyAwesomeStrategy(IStrategy): class HyperOpt: # Define a custom stoploss space. - def stoploss_space(self): + def stoploss_space(): return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')] ``` From 880474594e5357eea9b47b8d1c8449a968bce4c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 06:51:31 +0200 Subject: [PATCH 23/58] have ftuser use `/bin/bash` in dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4c4722452..f7e26efe3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ RUN mkdir /freqtrade \ && apt-get update \ && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ && apt-get clean \ - && useradd -u 1000 -G sudo -U -m ftuser \ + && useradd -u 1000 -G sudo -U -m -s /bin/bash ftuser \ && chown ftuser:ftuser /freqtrade \ # Allow sudoers && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers From 71ff214adfe844a1ff57624094169f48dbd202e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 07:14:40 +0200 Subject: [PATCH 24/58] Support "initial_call" for download-data of new pairs --- freqtrade/data/history/history_utils.py | 3 ++- freqtrade/exchange/binance.py | 1 + freqtrade/exchange/exchange.py | 17 ++++++++++++----- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 6f125aaa9..e6b8db322 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -197,7 +197,8 @@ def _download_pair_history(pair: str, *, timeframe=timeframe, since_ms=since_ms if since_ms else arrow.utcnow().shift( - days=-new_pairs_days).int_timestamp * 1000 + days=-new_pairs_days).int_timestamp * 1000, + is_new_pair=data.empty ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 189f5f481..7a3111f85 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -20,6 +20,7 @@ class Binance(Exchange): "order_time_in_force": ['gtc', 'fok', 'ioc'], "time_in_force_parameter": "timeInForce", "ohlcv_candle_limit": 1000, + "ohlcv_initial_call": True, "trades_pagination": "id", "trades_pagination_arg": "fromId", "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000], diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 80f20b17e..425b89f88 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1194,7 +1194,7 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int) -> List: + since_ms: int, is_new_pair: bool = False) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1206,7 +1206,7 @@ class Exchange: """ return asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms)) + since_ms=since_ms, is_new_pair=is_new_pair)) def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, since_ms: int) -> DataFrame: @@ -1221,9 +1221,9 @@ class Exchange: return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, - timeframe: str, - since_ms: int) -> List: + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: """ Download historic ohlcv """ @@ -1234,6 +1234,13 @@ class Exchange: one_call, arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) ) + if self._ft_has.get('ohlcv_initial_call', False) and is_new_pair: + x = await self._async_get_candle_history(pair, timeframe, 0) + if x and x[2] and x[2][0] and x[2][0][0] > since_ms: + # Set starting date to first available candle. + since_ms = x[2][0][0] + logger.info(f"Candle-data available starting with {since_ms}.") + input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] From 8c83c258a52c9ea4d91ed8f0d1a87e084109f828 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 19:29:10 +0200 Subject: [PATCH 25/58] Move "first-pair_getting" to binance subclass --- freqtrade/exchange/binance.py | 21 +++++++++++++++++++-- freqtrade/exchange/exchange.py | 8 +------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 7a3111f85..284f9cdad 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -1,7 +1,8 @@ """ Binance exchange subclass """ import logging -from typing import Dict +from typing import Dict, List +import arrow import ccxt from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, @@ -20,7 +21,6 @@ class Binance(Exchange): "order_time_in_force": ['gtc', 'fok', 'ioc'], "time_in_force_parameter": "timeInForce", "ohlcv_candle_limit": 1000, - "ohlcv_initial_call": True, "trades_pagination": "id", "trades_pagination_arg": "fromId", "l2_limit_range": [5, 10, 20, 50, 100, 500, 1000], @@ -91,3 +91,20 @@ class Binance(Exchange): f'Could not place sell order due to {e.__class__.__name__}. Message: {e}') from e except ccxt.BaseError as e: raise OperationalException(e) from e + + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool + ) -> List: + """ + Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date + Does not work for other exchanges, which don't return the earliest data when called with "0" + """ + if is_new_pair: + x = await self._async_get_candle_history(pair, timeframe, 0) + if x and x[2] and x[2][0] and x[2][0][0] > since_ms: + # Set starting date to first available candle. + since_ms = x[2][0][0] + logger.info(f"Candle-data available starting with " + f"{arrow.get(since_ms // 1000).isoformat()}.") + return await super()._async_get_historic_ohlcv( + pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 425b89f88..79fd33dfe 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1226,6 +1226,7 @@ class Exchange: ) -> List: """ Download historic ohlcv + :param is_new_pair: used by binance subclass to allow "fast" new pair downloading """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1234,13 +1235,6 @@ class Exchange: one_call, arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) ) - if self._ft_has.get('ohlcv_initial_call', False) and is_new_pair: - x = await self._async_get_candle_history(pair, timeframe, 0) - if x and x[2] and x[2][0] and x[2][0][0] > since_ms: - # Set starting date to first available candle. - since_ms = x[2][0][0] - logger.info(f"Candle-data available starting with {since_ms}.") - input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] From 2d66987ac7ed38961c2110680b03893693f86231 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Sep 2021 19:51:07 +0200 Subject: [PATCH 26/58] Add test for "pair-startdate" detection --- freqtrade/exchange/binance.py | 2 +- tests/exchange/test_binance.py | 35 +++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 284f9cdad..8dced3894 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -104,7 +104,7 @@ class Binance(Exchange): if x and x[2] and x[2][0] and x[2][0][0] > since_ms: # Set starting date to first available candle. since_ms = x[2][0][0] - logger.info(f"Candle-data available starting with " + logger.info(f"Candle-data for {pair} available starting with " f"{arrow.get(since_ms // 1000).isoformat()}.") return await super()._async_get_historic_ohlcv( pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair) diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index f2b508761..dd85c3abe 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -1,3 +1,4 @@ +from datetime import datetime, timezone from random import randint from unittest.mock import MagicMock @@ -5,7 +6,7 @@ import ccxt import pytest from freqtrade.exceptions import DependencyException, InvalidOrderException, OperationalException -from tests.conftest import get_patched_exchange +from tests.conftest import get_mock_coro, get_patched_exchange, log_has_re from tests.exchange.test_exchange import ccxt_exceptionhandlers @@ -105,3 +106,35 @@ def test_stoploss_adjust_binance(mocker, default_conf): # Test with invalid order case order['type'] = 'stop_loss' assert not exchange.stoploss_adjust(1501, order) + + +@pytest.mark.asyncio +async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog): + ohlcv = [ + [ + int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + + exchange = get_patched_exchange(mocker, default_conf, id='binance') + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) + + pair = 'ETH/BTC' + res = await exchange._async_get_historic_ohlcv(pair, "5m", + 1500000000000, is_new_pair=False) + # Call with very old timestamp - causes tons of requests + assert exchange._api_async.fetch_ohlcv.call_count > 400 + # assert res == ohlcv + exchange._api_async.fetch_ohlcv.reset_mock() + res = await exchange._async_get_historic_ohlcv(pair, "5m", 1500000000000, is_new_pair=True) + + # Called twice - one "init" call - and one to get the actual data. + assert exchange._api_async.fetch_ohlcv.call_count == 2 + assert res == ohlcv + assert log_has_re(r"Candle-data for ETH/BTC available starting with .*", caplog) From 79ca6135a2cb599de9fa4fde397226d63dda2832 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Tue, 7 Sep 2021 21:53:38 -0600 Subject: [PATCH 27/58] added caplog clears to freqtradebot tests --- tests/test_freqtradebot.py | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 75b67e59c..ddd031f77 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -107,6 +107,7 @@ def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: freqtrade = FreqtradeBot(conf) assert not freqtrade.strategy.order_types['stoploss_on_exchange'] assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) + caplog.clear() def test_order_dict_live(default_conf, mocker, caplog) -> None: @@ -140,6 +141,7 @@ def test_order_dict_live(default_conf, mocker, caplog) -> None: freqtrade = FreqtradeBot(conf) assert not freqtrade.strategy.order_types['stoploss_on_exchange'] assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) + caplog.clear() def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: @@ -415,6 +417,7 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord assert freqtrade.create_trade('ETH/BTC') assert log_has_re(r"Stake amount for pair .* is too small.*", caplog) + caplog.clear() def test_create_trade_zero_stake_amount(default_conf, ticker, limit_buy_order_open, @@ -478,6 +481,7 @@ def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_ope n = freqtrade.enter_positions() assert n == 0 assert log_has_re(r"No currency pair in active pair whitelist.*", caplog) + caplog.clear() def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, @@ -518,11 +522,13 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, # 0 trades, but it's not because of pairlock. assert n == 0 assert not log_has_re(message, caplog) + caplog.clear() PairLocks.lock_pair('*', arrow.utcnow().shift(minutes=20).datetime, 'Just because') n = freqtrade.enter_positions() assert n == 0 assert log_has_re(message, caplog) + caplog.clear() def test_create_trade_no_signal(default_conf, fee, mocker) -> None: @@ -1086,6 +1092,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, assert log_has_re(r'STOP_LOSS_LIMIT is hit for Trade\(id=1, .*\)\.', caplog) assert trade.stoploss_order_id is None assert trade.is_open is False + caplog.clear() mocker.patch( 'freqtrade.exchange.Binance.stoploss', @@ -1115,6 +1122,7 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) assert freqtrade.handle_stoploss_on_exchange(trade) is False assert stoploss.call_count == 0 + caplog.clear() def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, @@ -1154,6 +1162,7 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, assert log_has_re(r'Stoploss order was cancelled, but unable to recreate one.*', caplog) assert trade.stoploss_order_id is None assert trade.is_open is True + caplog.clear() def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, @@ -1202,6 +1211,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert rpc_mock.call_count == 2 assert rpc_mock.call_args_list[1][0][0]['sell_reason'] == SellType.EMERGENCY_SELL.value assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market' + caplog.clear() def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, fee, @@ -1428,6 +1438,7 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert cancel_mock.call_count == 1 assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) + caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -1662,6 +1673,7 @@ def test_enter_positions(mocker, default_conf, caplog) -> None: assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog) # create_trade should be called once for every pair in the whitelist. assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) + caplog.clear() def test_enter_positions_exception(mocker, default_conf, caplog) -> None: @@ -1701,6 +1713,7 @@ def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: # test amount modified by fee-logic n = freqtrade.exit_positions(trades) assert n == 0 + caplog.clear() def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1721,6 +1734,7 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) n = freqtrade.exit_positions(trades) assert n == 0 assert log_has('Unable to sell trade ETH/BTC: ', caplog) + caplog.clear() def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1743,10 +1757,12 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No ) assert not freqtrade.update_trade_state(trade, None) assert log_has_re(r'Orderid for trade .* is empty.', caplog) + caplog.clear() # Add datetime explicitly since sqlalchemy defaults apply only once written to database freqtrade.update_trade_state(trade, '123') # Test amount not modified by fee-logic assert not log_has_re(r'Applying fee to .*', caplog) + caplog.clear() assert trade.open_order_id is None assert trade.amount == limit_buy_order['amount'] @@ -1764,6 +1780,7 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No freqtrade.update_trade_state(trade, '123') assert log_has_re('Found open order for.*', caplog) + caplog.clear() def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, @@ -1814,6 +1831,7 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_ assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] assert log_has_re(r'Applying fee on amount for .*', caplog) + caplog.clear() def test_update_trade_state_exception(mocker, default_conf, @@ -1832,6 +1850,7 @@ def test_update_trade_state_exception(mocker, default_conf, ) freqtrade.update_trade_state(trade, trade.open_order_id) assert log_has('Could not update trade amount: ', caplog) + caplog.clear() def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None: @@ -1848,6 +1867,7 @@ def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None freqtrade.update_trade_state(trade, trade.open_order_id) assert grm_mock.call_count == 0 assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) + caplog.clear() def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_open, @@ -2016,6 +2036,7 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open, assert freqtrade.handle_trade(trade) assert log_has("ETH/BTC - Required profit reached. sell_type=SellType.ROI", caplog) + caplog.clear() def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open, @@ -2048,6 +2069,7 @@ def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open assert freqtrade.handle_trade(trade) assert log_has("ETH/BTC - Sell signal received. sell_type=SellType.SELL_SIGNAL", caplog) + caplog.clear() def test_close_trade(default_conf, ticker, limit_buy_order, limit_buy_order_open, limit_sell_order, @@ -2088,6 +2110,7 @@ def test_bot_loop_start_called_once(mocker, default_conf, caplog): assert log_has_re(r'Strategy caused the following exception.*', caplog) assert ftbot.strategy.bot_loop_start.call_count == 1 assert ftbot.strategy.analyze.call_count == 1 + caplog.clear() def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_order_old, open_trade, @@ -2202,6 +2225,7 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o nb_trades = len(trades) assert nb_trades == 0 assert log_has_re("Buy order cancelled on exchange for Trade.*", caplog) + caplog.clear() def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, open_trade, @@ -2336,6 +2360,7 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, assert rpc_mock.call_count == 1 assert open_trade.is_open is True assert log_has_re("Sell order cancelled on exchange for Trade.*", caplog) + caplog.clear() def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, @@ -2404,6 +2429,7 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap assert trades[0].open_order_id is None assert trades[0].fee_updated('buy') assert pytest.approx(trades[0].fee_open) == 0.001 + caplog.clear() def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee, @@ -2444,6 +2470,7 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, limit_buy_order_old_partial['remaining']) assert trades[0].open_order_id is None assert trades[0].fee_open == fee() + caplog.clear() def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocker, caplog) -> None: @@ -2472,6 +2499,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke f"{open_trade.open_date.strftime('%Y-%m-%d %H:%M:%S')}" r"\) due to Traceback \(most recent call last\):\n*", caplog) + caplog.clear() def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None: @@ -2515,6 +2543,7 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) + caplog.clear() @pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], @@ -2536,6 +2565,7 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, assert cancel_order_mock.call_count == 0 assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog) assert nofiy_mock.call_count == 1 + caplog.clear() @pytest.mark.parametrize('cancelorder', [ @@ -2905,6 +2935,7 @@ def test_execute_trade_exit_sloe_cancel_exception( sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS)) assert create_order_mock.call_count == 2 assert log_has('Could not cancel stoploss order abcd', caplog) + caplog.clear() def test_execute_trade_exit_with_stoploss_on_exchange(default_conf, ticker, fee, ticker_sell_up, @@ -3300,6 +3331,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ assert freqtrade.handle_trade(trade) is True assert log_has_re(r'.*Falling back to wallet-amount.', caplog) assert trade.amount != amnt + caplog.clear() def test__safe_sell_amount(default_conf, fee, caplog, mocker): @@ -3330,6 +3362,7 @@ def test__safe_sell_amount(default_conf, fee, caplog, mocker): assert freqtrade._safe_sell_amount(trade.pair, amount_wallet) == amount_wallet assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) assert wallet_update.call_count == 1 + caplog.clear() def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): @@ -3386,6 +3419,7 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo freqtrade.enter_positions() assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) + caplog.clear() def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, @@ -3477,6 +3511,7 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, assert log_has("ETH/BTC - HIT STOP: current price at 0.000012, stoploss is 0.000015, " "initial stoploss was at 0.000010, trade opened at 0.000011", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value + caplog.clear() def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3524,6 +3559,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0 profit: 0.2666%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 + caplog.clear() mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -3537,6 +3573,7 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or f"ETH/BTC - HIT STOP: current price at {buy_price + 0.000002:.6f}, " f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) + caplog.clear() def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3584,6 +3621,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde assert log_has("ETH/BTC - Using positive stoploss: 0.01 offset: 0.011 profit: 0.2666%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000138501 + caplog.clear() mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', MagicMock(return_value={ @@ -3598,6 +3636,7 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value + caplog.clear() def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3648,6 +3687,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ assert not log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000098910 + caplog.clear() # price rises above the offset (rises 12% when the offset is 5.5%) mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', @@ -3661,6 +3701,7 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ assert log_has("ETH/BTC - Using positive stoploss: 0.05 offset: 0.055 profit: 0.1218%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000117705 + caplog.clear() def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, @@ -3722,6 +3763,7 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) + caplog.clear() def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, @@ -3747,6 +3789,7 @@ def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fe assert walletmock.call_count == 1 assert log_has_re(r'Fee amount for Trade.* was in base currency ' '- Eating Fee 0.008 into dust', caplog) + caplog.clear() def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): @@ -3769,6 +3812,7 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) failed: myTrade-Dict empty found', caplog) + caplog.clear() def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): @@ -3862,6 +3906,7 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None + caplog.clear() def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee, @@ -3897,6 +3942,7 @@ def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None + caplog.clear() def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee, @@ -3925,6 +3971,7 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', caplog) + caplog.clear() def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): @@ -4168,6 +4215,7 @@ def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None with pytest.raises(PricingError): freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") assert log_has_re(r'Buy Price at location 1 from orderbook could not be determined.', caplog) + caplog.clear() def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: @@ -4238,6 +4286,7 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o freqtrade.handle_trade(trade) assert log_has_re(r'Sell Price at location 1 from orderbook could not be determined\..*', caplog) + caplog.clear() def test_startup_state(default_conf, mocker): @@ -4296,6 +4345,7 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_ assert log_has_re(r"Unable to create trade for XRP/BTC: " r"Available balance \(0.0 BTC\) is lower than stake amount \(0.001 BTC\)", caplog) + caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -4339,6 +4389,7 @@ def test_update_open_orders(mocker, default_conf, fee, caplog): freqtrade.update_open_orders() assert not log_has_re(r"Error updating Order .*", caplog) + caplog.clear() freqtrade.config['dry_run'] = False freqtrade.update_open_orders() @@ -4454,6 +4505,7 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 0 assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) + caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -4591,6 +4643,7 @@ def test_refind_lost_order(mocker, default_conf, fee, caplog): freqtrade.refind_lost_order(trades[4]) assert log_has(f"Error updating {order['id']}.", caplog) + caplog.clear() def test_get_valid_price(mocker, default_conf) -> None: From aac05029e1d13936f89c10f0ed8ccdaec3561ed7 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 01:20:52 -0600 Subject: [PATCH 28/58] safe_sell_amount -> safe_exit_amount --- freqtrade/freqtradebot.py | 4 ++-- tests/test_freqtradebot.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 259270483..0fc40cf45 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1039,7 +1039,7 @@ class FreqtradeBot(LoggingMixin): ) return reason - def _safe_sell_amount(self, pair: str, amount: float) -> float: + def _safe_exit_amount(self, pair: str, amount: float) -> float: """ Get sellable amount. Should be trade.amount - but will fall back to the available amount if necessary. @@ -1111,7 +1111,7 @@ class FreqtradeBot(LoggingMixin): # but we allow this value to be changed) order_type = self.strategy.order_types.get("forcesell", order_type) - amount = self._safe_sell_amount(trade.pair, trade.amount) + amount = self._safe_exit_amount(trade.pair, trade.amount) time_in_force = self.strategy.order_time_in_force['sell'] if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 75b67e59c..109cb01c2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -3302,7 +3302,7 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ assert trade.amount != amnt -def test__safe_sell_amount(default_conf, fee, caplog, mocker): +def test__safe_exit_amount(default_conf, fee, caplog, mocker): patch_RPCManager(mocker) patch_exchange(mocker) amount = 95.33 @@ -3322,17 +3322,17 @@ def test__safe_sell_amount(default_conf, fee, caplog, mocker): patch_get_signal(freqtrade) wallet_update.reset_mock() - assert freqtrade._safe_sell_amount(trade.pair, trade.amount) == amount_wallet + assert freqtrade._safe_exit_amount(trade.pair, trade.amount) == amount_wallet assert log_has_re(r'.*Falling back to wallet-amount.', caplog) assert wallet_update.call_count == 1 caplog.clear() wallet_update.reset_mock() - assert freqtrade._safe_sell_amount(trade.pair, amount_wallet) == amount_wallet + assert freqtrade._safe_exit_amount(trade.pair, amount_wallet) == amount_wallet assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) assert wallet_update.call_count == 1 -def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): +def test__safe_exit_amount_error(default_conf, fee, caplog, mocker): patch_RPCManager(mocker) patch_exchange(mocker) amount = 95.33 @@ -3350,7 +3350,7 @@ def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) with pytest.raises(DependencyException, match=r"Not enough amount to sell."): - assert freqtrade._safe_sell_amount(trade.pair, trade.amount) + assert freqtrade._safe_exit_amount(trade.pair, trade.amount) def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplog) -> None: From be93c75e44fcaa1df2094b1d6200b0c884abf596 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 01:14:16 -0600 Subject: [PATCH 29/58] reupdate_buy_order_fees -> reupdate_enter_order_fees --- freqtrade/freqtradebot.py | 4 ++-- tests/test_freqtradebot.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0fc40cf45..0f16162cd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -296,9 +296,9 @@ class FreqtradeBot(LoggingMixin): if sell_order: self.refind_lost_order(trade) else: - self.reupdate_buy_order_fees(trade) + self.reupdate_enter_order_fees(trade) - def reupdate_buy_order_fees(self, trade: Trade): + def reupdate_enter_order_fees(self, trade: Trade): """ Get buy order from database, and try to reupdate. Handles trades where the initial fee-update did not work. diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 109cb01c2..88ed11558 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -4420,14 +4420,14 @@ def test_update_closed_trades_without_assigned_fees(mocker, default_conf, fee): @pytest.mark.usefixtures("init_persistence") -def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): +def test_reupdate_enter_order_fees(mocker, default_conf, fee, caplog): freqtrade = get_patched_freqtradebot(mocker, default_conf) mock_uts = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.update_trade_state') create_mock_trades(fee) trades = Trade.get_trades().all() - freqtrade.reupdate_buy_order_fees(trades[0]) + freqtrade.reupdate_enter_order_fees(trades[0]) assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 1 assert mock_uts.call_args_list[0][0][0] == trades[0] @@ -4450,7 +4450,7 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): ) Trade.query.session.add(trade) - freqtrade.reupdate_buy_order_fees(trade) + freqtrade.reupdate_enter_order_fees(trade) assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 0 assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) @@ -4460,7 +4460,7 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): def test_handle_insufficient_funds(mocker, default_conf, fee): freqtrade = get_patched_freqtradebot(mocker, default_conf) mock_rlo = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.refind_lost_order') - mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_buy_order_fees') + mock_bof = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.reupdate_enter_order_fees') create_mock_trades(fee) trades = Trade.get_trades().all() From e0092a85e9769eeb7e73bbb6f6018dbd494ea1a0 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 00:53:09 -0600 Subject: [PATCH 30/58] handle_cancel_buy/sell -> handle_cancel_enter/exit --- freqtrade/freqtradebot.py | 12 +++++------ freqtrade/rpc/rpc.py | 4 ++-- tests/test_freqtradebot.py | 44 +++++++++++++++++++------------------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0f16162cd..bf07ac0bc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -906,7 +906,7 @@ class FreqtradeBot(LoggingMixin): default_retval=False)(pair=trade.pair, trade=trade, order=order))): - self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['TIMEOUT']) + self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['TIMEOUT']) elif (order['side'] == 'sell' and (order['status'] == 'open' or fully_cancelled) and ( fully_cancelled @@ -915,7 +915,7 @@ class FreqtradeBot(LoggingMixin): default_retval=False)(pair=trade.pair, trade=trade, order=order))): - self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['TIMEOUT']) + self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['TIMEOUT']) def cancel_all_open_orders(self) -> None: """ @@ -931,13 +931,13 @@ class FreqtradeBot(LoggingMixin): continue if order['side'] == 'buy': - self.handle_cancel_buy(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + self.handle_cancel_enter(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) elif order['side'] == 'sell': - self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + self.handle_cancel_exit(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) Trade.commit() - def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool: + def handle_cancel_enter(self, trade: Trade, order: Dict, reason: str) -> bool: """ Buy cancel - cancel order :return: True if order was fully cancelled @@ -998,7 +998,7 @@ class FreqtradeBot(LoggingMixin): reason=reason) return was_trade_fully_canceled - def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str: + def handle_cancel_exit(self, trade: Trade, order: Dict, reason: str) -> str: """ Sell cancel - cancel order and update trade :return: Reason for cancel diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index ca2e84e48..caf7345a2 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -548,12 +548,12 @@ class RPC: order = self._freqtrade.exchange.fetch_order(trade.open_order_id, trade.pair) if order['side'] == 'buy': - fully_canceled = self._freqtrade.handle_cancel_buy( + fully_canceled = self._freqtrade.handle_cancel_enter( trade, order, CANCEL_REASON['FORCE_SELL']) if order['side'] == 'sell': # Cancel order - so it is placed anew with a fresh price. - self._freqtrade.handle_cancel_sell(trade, order, CANCEL_REASON['FORCE_SELL']) + self._freqtrade.handle_cancel_exit(trade, order, CANCEL_REASON['FORCE_SELL']) if not fully_canceled: # Get current rate and execute sell diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 88ed11558..1b5381264 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2453,8 +2453,8 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', - handle_cancel_buy=MagicMock(), - handle_cancel_sell=MagicMock(), + handle_cancel_enter=MagicMock(), + handle_cancel_exit=MagicMock(), ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -2474,7 +2474,7 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke caplog) -def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None: +def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_buy_order = deepcopy(limit_buy_order) @@ -2493,34 +2493,34 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non limit_buy_order['filled'] = 0.0 limit_buy_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() caplog.clear() limit_buy_order['filled'] = 0.01 - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 0 assert log_has_re("Order .* for .* not cancelled, as the filled amount.* unsellable.*", caplog) caplog.clear() cancel_order_mock.reset_mock() limit_buy_order['filled'] = 2 - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 # Order remained open for some reason (cancel failed) cancel_buy_order['status'] = 'open' cancel_order_mock = MagicMock(return_value=cancel_buy_order) mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) @pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], indirect=['limit_buy_order_canceled_empty']) -def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, - limit_buy_order_canceled_empty) -> None: +def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf, + limit_buy_order_canceled_empty) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = mocker.patch( @@ -2532,7 +2532,7 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, reason = CANCEL_REASON['TIMEOUT'] trade = MagicMock() trade.pair = 'LTC/ETH' - assert freqtrade.handle_cancel_buy(trade, limit_buy_order_canceled_empty, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order_canceled_empty, reason) assert cancel_order_mock.call_count == 0 assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog) assert nofiy_mock.call_count == 1 @@ -2544,8 +2544,8 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, 'String Return value', 123 ]) -def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order, - cancelorder) -> None: +def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order, + cancelorder) -> None: patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock(return_value=cancelorder) @@ -2563,16 +2563,16 @@ def test_handle_cancel_buy_corder_empty(mocker, default_conf, limit_buy_order, limit_buy_order['filled'] = 0.0 limit_buy_order['status'] = 'open' reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 cancel_order_mock.reset_mock() limit_buy_order['filled'] = 1.0 - assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) + assert not freqtrade.handle_cancel_enter(trade, limit_buy_order, reason) assert cancel_order_mock.call_count == 1 -def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: +def test_handle_cancel_exit_limit(mocker, default_conf, fee) -> None: send_msg_mock = patch_RPCManager(mocker) patch_exchange(mocker) cancel_order_mock = MagicMock() @@ -2598,26 +2598,26 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: 'amount': 1, 'status': "open"} reason = CANCEL_REASON['TIMEOUT'] - assert freqtrade.handle_cancel_sell(trade, order, reason) + assert freqtrade.handle_cancel_exit(trade, order, reason) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 send_msg_mock.reset_mock() order['amount'] = 2 - assert freqtrade.handle_cancel_sell(trade, order, reason + assert freqtrade.handle_cancel_exit(trade, order, reason ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] # Assert cancel_order was not called (callcount remains unchanged) assert cancel_order_mock.call_count == 1 assert send_msg_mock.call_count == 1 - assert freqtrade.handle_cancel_sell(trade, order, reason + assert freqtrade.handle_cancel_exit(trade, order, reason ) == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] # Message should not be iterated again assert trade.sell_order_status == CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] assert send_msg_mock.call_count == 1 -def test_handle_cancel_sell_cancel_exception(mocker, default_conf) -> None: +def test_handle_cancel_exit_cancel_exception(mocker, default_conf) -> None: patch_RPCManager(mocker) patch_exchange(mocker) mocker.patch( @@ -2630,7 +2630,7 @@ def test_handle_cancel_sell_cancel_exception(mocker, default_conf) -> None: order = {'remaining': 1, 'amount': 1, 'status': "open"} - assert freqtrade.handle_cancel_sell(trade, order, reason) == 'error cancelling order' + assert freqtrade.handle_cancel_exit(trade, order, reason) == 'error cancelling order' def test_execute_trade_exit_up(default_conf, ticker, fee, ticker_sell_up, mocker) -> None: @@ -4304,8 +4304,8 @@ def test_cancel_all_open_orders(mocker, default_conf, fee, limit_buy_order, limi mocker.patch('freqtrade.exchange.Exchange.fetch_order', side_effect=[ ExchangeError(), limit_sell_order, limit_buy_order, limit_sell_order]) - buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_buy') - sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_sell') + buy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_enter') + sell_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot.handle_cancel_exit') freqtrade = get_patched_freqtradebot(mocker, default_conf) create_mock_trades(fee) From e1f846f22f28a9f76df1e8c47a79225ca2bc6e89 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 00:49:04 -0600 Subject: [PATCH 31/58] sell_lock -> exit_lock --- freqtrade/freqtradebot.py | 6 +++--- freqtrade/rpc/rpc.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index bf07ac0bc..0a0b20f0c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -99,7 +99,7 @@ class FreqtradeBot(LoggingMixin): self.state = State[initial_state.upper()] if initial_state else State.STOPPED # Protect sell-logic from forcesell and vice versa - self._sell_lock = Lock() + self._exit_lock = Lock() LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) def notify_status(self, msg: str) -> None: @@ -166,14 +166,14 @@ class FreqtradeBot(LoggingMixin): self.strategy.analyze(self.active_pair_whitelist) - with self._sell_lock: + with self._exit_lock: # Check and handle any timed out open orders self.check_handle_timedout() # Protect from collisions with forcesell. # Without this, freqtrade my try to recreate stoploss_on_exchange orders # while selling is in process, since telegram messages arrive in an different thread. - with self._sell_lock: + with self._exit_lock: trades = Trade.get_open_trades() # First process current opened trades (positions) self.exit_positions(trades) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index caf7345a2..b7b1fe603 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -566,7 +566,7 @@ class RPC: if self._freqtrade.state != State.RUNNING: raise RPCException('trader is not running') - with self._freqtrade._sell_lock: + with self._freqtrade._exit_lock: if trade_id == 'all': # Execute sell for all open orders for trade in Trade.get_open_trades(): @@ -628,7 +628,7 @@ class RPC: Handler for delete . Delete the given trade and close eventually existing open orders. """ - with self._freqtrade._sell_lock: + with self._freqtrade._exit_lock: c_count = 0 trade = Trade.get_trades(trade_filter=[Trade.id == trade_id]).first() if not trade: From 362dc20406c0419246cbe459518636337567cf9b Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 00:45:55 -0600 Subject: [PATCH 32/58] notify_buy -> notify_enter, notify_sell -> notify_exit --- freqtrade/freqtradebot.py | 26 +++++++++++++------------- tests/test_freqtradebot.py | 6 +++--- tests/test_integration.py | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 0a0b20f0c..b67ae9f00 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -590,11 +590,11 @@ class FreqtradeBot(LoggingMixin): # Updating wallets self.wallets.update() - self._notify_buy(trade, order_type) + self._notify_enter(trade, order_type) return True - def _notify_buy(self, trade: Trade, order_type: str) -> None: + def _notify_enter(self, trade: Trade, order_type: str) -> None: """ Sends rpc notification when a buy occurred. """ @@ -617,7 +617,7 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None: + def _notify_enter_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a buy cancel occurred. """ @@ -643,7 +643,7 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_buy_fill(self, trade: Trade) -> None: + def _notify_enter_fill(self, trade: Trade) -> None: msg = { 'trade_id': trade.id, 'type': RPCMessageType.BUY_FILL, @@ -782,7 +782,7 @@ class FreqtradeBot(LoggingMixin): # Lock pair for one candle to prevent immediate rebuys self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') - self._notify_sell(trade, "stoploss") + self._notify_exit(trade, "stoploss") return True if trade.open_order_id or not trade.is_open: @@ -994,8 +994,8 @@ class FreqtradeBot(LoggingMixin): reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}" self.wallets.update() - self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'], - reason=reason) + self._notify_enter_cancel(trade, order_type=self.strategy.order_types['buy'], + reason=reason) return was_trade_fully_canceled def handle_cancel_exit(self, trade: Trade, order: Dict, reason: str) -> str: @@ -1032,7 +1032,7 @@ class FreqtradeBot(LoggingMixin): reason = constants.CANCEL_REASON['PARTIALLY_FILLED_KEEP_OPEN'] self.wallets.update() - self._notify_sell_cancel( + self._notify_exit_cancel( trade, order_type=self.strategy.order_types['sell'], reason=reason @@ -1150,11 +1150,11 @@ class FreqtradeBot(LoggingMixin): self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') - self._notify_sell(trade, order_type) + self._notify_exit(trade, order_type) return True - def _notify_sell(self, trade: Trade, order_type: str, fill: bool = False) -> None: + def _notify_exit(self, trade: Trade, order_type: str, fill: bool = False) -> None: """ Sends rpc notification when a sell occurred. """ @@ -1196,7 +1196,7 @@ class FreqtradeBot(LoggingMixin): # Send the message self.rpc.send_msg(msg) - def _notify_sell_cancel(self, trade: Trade, order_type: str, reason: str) -> None: + def _notify_exit_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ Sends rpc notification when a sell cancel occurred. """ @@ -1291,13 +1291,13 @@ class FreqtradeBot(LoggingMixin): # Updating wallets when order is closed if not trade.is_open: if not stoploss_order and not trade.open_order_id: - self._notify_sell(trade, '', True) + self._notify_exit(trade, '', True) self.protections.stop_per_pair(trade.pair) self.protections.global_stop() self.wallets.update() elif not trade.open_order_id: # Buy fill - self._notify_buy_fill(trade) + self._notify_enter_fill(trade) return False diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 1b5381264..26d06d53a 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -2485,7 +2485,7 @@ def test_handle_cancel_enter(mocker, caplog, default_conf, limit_buy_order) -> N mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) freqtrade = FreqtradeBot(default_conf) - freqtrade._notify_buy_cancel = MagicMock() + freqtrade._notify_enter_cancel = MagicMock() trade = MagicMock() trade.pair = 'LTC/USDT' @@ -2526,7 +2526,7 @@ def test_handle_cancel_enter_exchanges(mocker, caplog, default_conf, cancel_order_mock = mocker.patch( 'freqtrade.exchange.Exchange.cancel_order_with_result', return_value=limit_buy_order_canceled_empty) - nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_buy_cancel') + nofiy_mock = mocker.patch('freqtrade.freqtradebot.FreqtradeBot._notify_enter_cancel') freqtrade = FreqtradeBot(default_conf) reason = CANCEL_REASON['TIMEOUT'] @@ -2555,7 +2555,7 @@ def test_handle_cancel_enter_corder_empty(mocker, default_conf, limit_buy_order, ) freqtrade = FreqtradeBot(default_conf) - freqtrade._notify_buy_cancel = MagicMock() + freqtrade._notify_enter_cancel = MagicMock() trade = MagicMock() trade.pair = 'LTC/USDT' diff --git a/tests/test_integration.py b/tests/test_integration.py index 215927098..a3484d438 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -70,7 +70,7 @@ def test_may_execute_exit_stoploss_on_exchange_multi(default_conf, ticker, fee, mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - _notify_sell=MagicMock(), + _notify_exit=MagicMock(), ) mocker.patch("freqtrade.strategy.interface.IStrategy.should_sell", should_sell_mock) wallets_mock = mocker.patch("freqtrade.wallets.Wallets.update", MagicMock()) @@ -154,7 +154,7 @@ def test_forcebuy_last_unlimited(default_conf, ticker, fee, limit_buy_order, moc mocker.patch.multiple( 'freqtrade.freqtradebot.FreqtradeBot', create_stoploss_order=MagicMock(return_value=True), - _notify_sell=MagicMock(), + _notify_exit=MagicMock(), ) should_sell_mock = MagicMock(side_effect=[ SellCheckTuple(sell_type=SellType.NONE), From a1c9a4d619f3dfe5f123e928487f5dcdf1d351fc Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 01:40:22 -0600 Subject: [PATCH 33/58] freqtradebot local name changes --- freqtrade/freqtradebot.py | 48 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index b67ae9f00..5800befba 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -476,21 +476,21 @@ class FreqtradeBot(LoggingMixin): time_in_force = self.strategy.order_time_in_force['buy'] if price: - buy_limit_requested = price + enter_limit_requested = price else: # Calculate price - proposed_buy_rate = self.exchange.get_rate(pair, refresh=True, side="buy") + proposed_enter_rate = self.exchange.get_rate(pair, refresh=True, side="buy") custom_entry_price = strategy_safe_wrapper(self.strategy.custom_entry_price, - default_retval=proposed_buy_rate)( + default_retval=proposed_enter_rate)( pair=pair, current_time=datetime.now(timezone.utc), - proposed_rate=proposed_buy_rate) + proposed_rate=proposed_enter_rate) - buy_limit_requested = self.get_valid_price(custom_entry_price, proposed_buy_rate) + enter_limit_requested = self.get_valid_price(custom_entry_price, proposed_enter_rate) - if not buy_limit_requested: + if not enter_limit_requested: raise PricingError('Could not determine buy price.') - min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, buy_limit_requested, + min_stake_amount = self.exchange.get_min_pair_stake_amount(pair, enter_limit_requested, self.strategy.stoploss) if not self.edge: @@ -498,7 +498,7 @@ class FreqtradeBot(LoggingMixin): stake_amount = strategy_safe_wrapper(self.strategy.custom_stake_amount, default_retval=stake_amount)( pair=pair, current_time=datetime.now(timezone.utc), - current_rate=buy_limit_requested, proposed_stake=stake_amount, + current_rate=enter_limit_requested, proposed_stake=stake_amount, min_stake=min_stake_amount, max_stake=max_stake_amount) stake_amount = self.wallets._validate_stake_amount(pair, stake_amount, min_stake_amount) @@ -508,27 +508,27 @@ class FreqtradeBot(LoggingMixin): logger.info(f"Buy signal found: about create a new trade for {pair} with stake_amount: " f"{stake_amount} ...") - amount = stake_amount / buy_limit_requested + amount = stake_amount / enter_limit_requested order_type = self.strategy.order_types['buy'] if forcebuy: # Forcebuy can define a different ordertype order_type = self.strategy.order_types.get('forcebuy', order_type) if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( - pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested, + pair=pair, order_type=order_type, amount=amount, rate=enter_limit_requested, time_in_force=time_in_force, current_time=datetime.now(timezone.utc)): logger.info(f"User requested abortion of buying {pair}") return False amount = self.exchange.amount_to_precision(pair, amount) order = self.exchange.create_order(pair=pair, ordertype=order_type, side="buy", - amount=amount, rate=buy_limit_requested, + amount=amount, rate=enter_limit_requested, time_in_force=time_in_force) order_obj = Order.parse_from_ccxt_object(order, pair, 'buy') order_id = order['id'] order_status = order.get('status', None) # we assume the order is executed at the price requested - buy_limit_filled_price = buy_limit_requested + enter_limit_filled_price = enter_limit_requested amount_requested = amount if order_status == 'expired' or order_status == 'rejected': @@ -551,13 +551,13 @@ class FreqtradeBot(LoggingMixin): ) stake_amount = order['cost'] amount = safe_value_fallback(order, 'filled', 'amount') - buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') + enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') # in case of FOK the order may be filled immediately and fully elif order_status == 'closed': stake_amount = order['cost'] amount = safe_value_fallback(order, 'filled', 'amount') - buy_limit_filled_price = safe_value_fallback(order, 'average', 'price') + enter_limit_filled_price = safe_value_fallback(order, 'average', 'price') # Fee is applied twice because we make a LIMIT_BUY and LIMIT_SELL fee = self.exchange.get_fee(symbol=pair, taker_or_maker='maker') @@ -569,8 +569,8 @@ class FreqtradeBot(LoggingMixin): amount_requested=amount_requested, fee_open=fee, fee_close=fee, - open_rate=buy_limit_filled_price, - open_rate_requested=buy_limit_requested, + open_rate=enter_limit_filled_price, + open_rate_requested=enter_limit_requested, open_date=datetime.utcnow(), exchange=self.exchange.id, open_order_id=order_id, @@ -713,8 +713,8 @@ class FreqtradeBot(LoggingMixin): ) logger.debug('checking sell') - sell_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell") - if self._check_and_execute_sell(trade, sell_rate, buy, sell): + exit_rate = self.exchange.get_rate(trade.pair, refresh=True, side="sell") + if self._check_and_execute_exit(trade, exit_rate, buy, sell): return True logger.debug('Found no sell signal for %s.', trade) @@ -744,7 +744,7 @@ class FreqtradeBot(LoggingMixin): except InvalidOrderException as e: trade.stoploss_order_id = None logger.error(f'Unable to place a stoploss order on exchange. {e}') - logger.warning('Selling the trade forcefully') + logger.warning('Exiting the trade forcefully') self.execute_trade_exit(trade, trade.stop_loss, sell_reason=SellCheckTuple( sell_type=SellType.EMERGENCY_SELL)) @@ -851,19 +851,19 @@ class FreqtradeBot(LoggingMixin): logger.warning(f"Could not create trailing stoploss order " f"for pair {trade.pair}.") - def _check_and_execute_sell(self, trade: Trade, sell_rate: float, - buy: bool, sell: bool) -> bool: + def _check_and_execute_exit(self, trade: Trade, exit_rate: float, + enter: bool, exit_: bool) -> bool: """ - Check and execute sell + Check and execute exit """ should_sell = self.strategy.should_sell( - trade, sell_rate, datetime.now(timezone.utc), buy, sell, + trade, exit_rate, datetime.now(timezone.utc), buy, sell, force_stoploss=self.edge.stoploss(trade.pair) if self.edge else 0 ) if should_sell.sell_flag: logger.info(f'Executing Sell for {trade.pair}. Reason: {should_sell.sell_type}') - self.execute_trade_exit(trade, sell_rate, should_sell) + self.execute_trade_exit(trade, exit_rate, should_sell) return True return False From b2f289e4040ae676830987c3436ba85f8b29667d Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 02:16:25 -0600 Subject: [PATCH 34/58] Fixed freqtradebot failing tests --- freqtrade/freqtradebot.py | 2 +- tests/test_freqtradebot.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 5800befba..7f668273c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -852,7 +852,7 @@ class FreqtradeBot(LoggingMixin): f"for pair {trade.pair}.") def _check_and_execute_exit(self, trade: Trade, exit_rate: float, - enter: bool, exit_: bool) -> bool: + buy: bool, sell: bool) -> bool: """ Check and execute exit """ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 26d06d53a..3c5a8cfae 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1190,7 +1190,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert trade.stoploss_order_id is None assert trade.sell_reason == SellType.EMERGENCY_SELL.value assert log_has("Unable to place a stoploss order on exchange. ", caplog) - assert log_has("Selling the trade forcefully", caplog) + assert log_has("Exiting the trade forcefully", caplog) # Should call a market sell assert create_order_mock.call_count == 2 From 1379ec74022888a0dd018470cb23aca0ab84a808 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 01:48:22 -0600 Subject: [PATCH 35/58] Updated log messages for freqtradebot --- freqtrade/freqtradebot.py | 6 +++--- tests/test_freqtradebot.py | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 53ca2764b..f2e8e3aa0 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -385,7 +385,7 @@ class FreqtradeBot(LoggingMixin): logger.warning('Unable to create trade for %s: %s', pair, exception) if not trades_created: - logger.debug("Found no buy signals for whitelisted currencies. Trying again...") + logger.debug("Found no enter signals for whitelisted currencies. Trying again...") return trades_created @@ -681,7 +681,7 @@ class FreqtradeBot(LoggingMixin): trades_closed += 1 except DependencyException as exception: - logger.warning('Unable to sell trade %s: %s', trade.pair, exception) + logger.warning('Unable to exit trade %s: %s', trade.pair, exception) # Updating wallets if any trade occurred if trades_closed: @@ -1062,7 +1062,7 @@ class FreqtradeBot(LoggingMixin): return wallet_amount else: raise DependencyException( - f"Not enough amount to sell. Trade-amount: {amount}, Wallet: {wallet_amount}") + f"Not enough amount to exit trade. Trade-amount: {amount}, Wallet: {wallet_amount}") def execute_trade_exit(self, trade: Trade, limit: float, sell_reason: SellCheckTuple) -> bool: """ diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 3432c34f6..fc4b6fb74 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1190,7 +1190,7 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert trade.stoploss_order_id is None assert trade.sell_reason == SellType.EMERGENCY_SELL.value assert log_has("Unable to place a stoploss order on exchange. ", caplog) - assert log_has("Selling the trade forcefully", caplog) + assert log_has("Exiting the trade forcefully", caplog) # Should call a market sell assert create_order_mock.call_count == 2 @@ -1659,7 +1659,7 @@ def test_enter_positions(mocker, default_conf, caplog) -> None: MagicMock(return_value=False)) n = freqtrade.enter_positions() assert n == 0 - assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog) + assert log_has('Found no enter signals for whitelisted currencies. Trying again...', caplog) # create_trade should be called once for every pair in the whitelist. assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) @@ -1720,7 +1720,8 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) ) n = freqtrade.exit_positions(trades) assert n == 0 - assert log_has('Unable to sell trade ETH/BTC: ', caplog) + assert log_has('Unable to exit trade ETH/BTC: ', caplog) + caplog.clear() def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -3350,7 +3351,7 @@ def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): ) freqtrade = FreqtradeBot(default_conf) patch_get_signal(freqtrade) - with pytest.raises(DependencyException, match=r"Not enough amount to sell."): + with pytest.raises(DependencyException, match=r"Not enough amount to exit."): assert freqtrade._safe_sell_amount(trade.pair, trade.amount) From 695a8fc73b92e9355e32a5c5ea7aea65db8bbed7 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Wed, 8 Sep 2021 01:53:42 -0600 Subject: [PATCH 36/58] comment updates, formatting, TODOs --- freqtrade/enums/signaltype.py | 2 +- freqtrade/freqtradebot.py | 86 +++++++++++++++++----------- freqtrade/optimize/backtesting.py | 2 +- freqtrade/persistence/models.py | 2 +- freqtrade/plugins/pairlistmanager.py | 2 +- freqtrade/rpc/rpc.py | 1 + tests/test_persistence.py | 2 +- 7 files changed, 58 insertions(+), 39 deletions(-) diff --git a/freqtrade/enums/signaltype.py b/freqtrade/enums/signaltype.py index d2995d57a..fc57e1ce7 100644 --- a/freqtrade/enums/signaltype.py +++ b/freqtrade/enums/signaltype.py @@ -3,7 +3,7 @@ from enum import Enum class SignalType(Enum): """ - Enum to distinguish between buy and sell signals + Enum to distinguish between enter and exit signals """ BUY = "buy" SELL = "sell" diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f2e8e3aa0..ec2745b03 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -66,6 +66,7 @@ class FreqtradeBot(LoggingMixin): init_db(self.config.get('db_url', None), clean_open_orders=self.config['dry_run']) + # TODO-lev: Do anything with this? self.wallets = Wallets(self.config, self.exchange) PairLocks.timeframe = self.config['timeframe'] @@ -77,6 +78,7 @@ class FreqtradeBot(LoggingMixin): # so anything in the Freqtradebot instance should be ready (initialized), including # the initial state of the bot. # Keep this at the end of this initialization method. + # TODO-lev: Do I need to consider the rpc, pairlists or dataprovider? self.rpc: RPCManager = RPCManager(self) self.pairlists = PairListManager(self.exchange, self.config) @@ -98,7 +100,7 @@ class FreqtradeBot(LoggingMixin): initial_state = self.config.get('initial_state') self.state = State[initial_state.upper()] if initial_state else State.STOPPED - # Protect sell-logic from forcesell and vice versa + # Protect exit-logic from forcesell and vice versa self._sell_lock = Lock() LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) @@ -170,9 +172,9 @@ class FreqtradeBot(LoggingMixin): # Check and handle any timed out open orders self.check_handle_timedout() - # Protect from collisions with forcesell. + # Protect from collisions with forceexit. # Without this, freqtrade my try to recreate stoploss_on_exchange orders - # while selling is in process, since telegram messages arrive in an different thread. + # while exiting is in process, since telegram messages arrive in an different thread. with self._sell_lock: trades = Trade.get_open_trades() # First process current opened trades (positions) @@ -289,8 +291,8 @@ class FreqtradeBot(LoggingMixin): def handle_insufficient_funds(self, trade: Trade): """ - Determine if we ever opened a sell order for this trade. - If not, try update buy fees - otherwise "refind" the open order we obviously lost. + Determine if we ever opened a exiting order for this trade. + If not, try update entering fees - otherwise "refind" the open order we obviously lost. """ sell_order = trade.select_order('sell', None) if sell_order: @@ -312,7 +314,7 @@ class FreqtradeBot(LoggingMixin): def refind_lost_order(self, trade): """ Try refinding a lost trade. - Only used when InsufficientFunds appears on sell orders (stoploss or sell). + Only used when InsufficientFunds appears on exit orders (stoploss or long sell/short buy). Tries to walk the stored orders and sell them off eventually. """ logger.info(f"Trying to refind lost order for {trade}") @@ -323,7 +325,7 @@ class FreqtradeBot(LoggingMixin): logger.debug(f"Order {order} is no longer open.") continue if order.ft_order_side == 'buy': - # Skip buy side - this is handled by reupdate_buy_order_fees + # Skip buy side - this is handled by reupdate_enter_order_fees continue try: fo = self.exchange.fetch_order_or_stoploss_order(order.order_id, order.ft_pair, @@ -350,7 +352,7 @@ class FreqtradeBot(LoggingMixin): def enter_positions(self) -> int: """ - Tries to execute buy orders for new trades (positions) + Tries to execute long buy/short sell orders for new trades (positions) """ trades_created = 0 @@ -366,7 +368,7 @@ class FreqtradeBot(LoggingMixin): if not whitelist: logger.info("No currency pair in active pair whitelist, " - "but checking to sell open trades.") + "but checking to exit open trades.") return trades_created if PairLocks.is_global_lock(): lock = PairLocks.get_pair_longest_lock('*') @@ -512,7 +514,9 @@ class FreqtradeBot(LoggingMixin): order_type = self.strategy.order_types['buy'] if forcebuy: # Forcebuy can define a different ordertype + # TODO-lev: get a forceshort? What is this order_type = self.strategy.order_types.get('forcebuy', order_type) + # TODO-lev: Will this work for shorting? if not strategy_safe_wrapper(self.strategy.confirm_trade_entry, default_retval=True)( pair=pair, order_type=order_type, amount=amount, rate=buy_limit_requested, @@ -596,7 +600,7 @@ class FreqtradeBot(LoggingMixin): def _notify_buy(self, trade: Trade, order_type: str) -> None: """ - Sends rpc notification when a buy occurred. + Sends rpc notification when a buy/short occurred. """ msg = { 'trade_id': trade.id, @@ -619,7 +623,7 @@ class FreqtradeBot(LoggingMixin): def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ - Sends rpc notification when a buy cancel occurred. + Sends rpc notification when a buy/short cancel occurred. """ current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="buy") @@ -665,7 +669,7 @@ class FreqtradeBot(LoggingMixin): def exit_positions(self, trades: List[Any]) -> int: """ - Tries to execute sell orders for open trades (positions) + Tries to execute sell/exit_short orders for open trades (positions) """ trades_closed = 0 for trade in trades: @@ -691,8 +695,8 @@ class FreqtradeBot(LoggingMixin): def handle_trade(self, trade: Trade) -> bool: """ - Sells the current pair if the threshold is reached and updates the trade record. - :return: True if trade has been sold, False otherwise + Sells/exits_short the current pair if the threshold is reached and updates the trade record. + :return: True if trade has been sold/exited_short, False otherwise """ if not trade.is_open: raise DependencyException(f'Attempt to handle closed trade: {trade}') @@ -700,7 +704,7 @@ class FreqtradeBot(LoggingMixin): logger.debug('Handling %s ...', trade) (buy, sell) = (False, False) - + # TODO-lev: change to use_exit_signal, ignore_roi_if_enter_signal if (self.config.get('use_sell_signal', True) or self.config.get('ignore_roi_if_buy_signal', False)): analyzed_df, _ = self.dataprovider.get_analyzed_dataframe(trade.pair, @@ -744,7 +748,7 @@ class FreqtradeBot(LoggingMixin): except InvalidOrderException as e: trade.stoploss_order_id = None logger.error(f'Unable to place a stoploss order on exchange. {e}') - logger.warning('Selling the trade forcefully') + logger.warning('Exiting the trade forcefully') self.execute_trade_exit(trade, trade.stop_loss, sell_reason=SellCheckTuple( sell_type=SellType.EMERGENCY_SELL)) @@ -758,6 +762,8 @@ class FreqtradeBot(LoggingMixin): Check if trade is fulfilled in which case the stoploss on exchange should be added immediately if stoploss on exchange is enabled. + # TODO-lev: liquidation price will always be on exchange, even though + # TODO-lev: stoploss_on_exchange might not be enabled """ logger.debug('Handling stoploss on exchange %s ...', trade) @@ -776,6 +782,7 @@ class FreqtradeBot(LoggingMixin): # We check if stoploss order is fulfilled if stoploss_order and stoploss_order['status'] in ('closed', 'triggered'): + # TODO-lev: Update to exit reason trade.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value self.update_trade_state(trade, trade.stoploss_order_id, stoploss_order, stoploss_order=True) @@ -791,7 +798,7 @@ class FreqtradeBot(LoggingMixin): # The trade can be closed already (sell-order fill confirmation came in this iteration) return False - # If buy order is fulfilled but there is no stoploss, we add a stoploss on exchange + # If enter order is fulfilled but there is no stoploss, we add a stoploss on exchange if not stoploss_order: stoploss = self.edge.stoploss(pair=trade.pair) if self.edge else self.strategy.stoploss stop_price = trade.open_rate * (1 + stoploss) @@ -942,6 +949,7 @@ class FreqtradeBot(LoggingMixin): Buy cancel - cancel order :return: True if order was fully cancelled """ + # TODO-lev: Pay back borrowed/interest and transfer back on leveraged trades was_trade_fully_canceled = False # Cancelled orders may have the status of 'canceled' or 'closed' @@ -986,6 +994,8 @@ class FreqtradeBot(LoggingMixin): # to the order dict acquired before cancelling. # we need to fall back to the values from order if corder does not contain these keys. trade.amount = filled_amount + # TODO-lev: Check edge cases, we don't want to make leverage > 1.0 if we don't have to + trade.stake_amount = trade.amount * trade.open_rate self.update_trade_state(trade, trade.open_order_id, corder) @@ -994,13 +1004,15 @@ class FreqtradeBot(LoggingMixin): reason += f", {constants.CANCEL_REASON['PARTIALLY_FILLED']}" self.wallets.update() + # TODO-lev: Should short and exit_short be an order type? + self._notify_buy_cancel(trade, order_type=self.strategy.order_types['buy'], reason=reason) return was_trade_fully_canceled def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str: """ - Sell cancel - cancel order and update trade + Sell/exit_short cancel - cancel order and update trade :return: Reason for cancel """ # if trade is not partially completed, just cancel the order @@ -1050,6 +1062,7 @@ class FreqtradeBot(LoggingMixin): :return: amount to sell :raise: DependencyException: if available balance is not within 2% of the available amount. """ + # TODO-lev Maybe update? # Update wallets to ensure amounts tied up in a stoploss is now free! self.wallets.update() trade_base_currency = self.exchange.get_pair_base_currency(pair) @@ -1072,7 +1085,7 @@ class FreqtradeBot(LoggingMixin): :param sell_reason: Reason the sell was triggered :return: True if it succeeds (supported) False (not supported) """ - sell_type = 'sell' + sell_type = 'sell' # TODO-lev: Update to exit if sell_reason.sell_type in (SellType.STOP_LOSS, SellType.TRAILING_STOP_LOSS): sell_type = 'stoploss' @@ -1112,22 +1125,25 @@ class FreqtradeBot(LoggingMixin): order_type = self.strategy.order_types.get("forcesell", order_type) amount = self._safe_sell_amount(trade.pair, trade.amount) - time_in_force = self.strategy.order_time_in_force['sell'] + time_in_force = self.strategy.order_time_in_force['sell'] # TODO-lev update to exit if not strategy_safe_wrapper(self.strategy.confirm_trade_exit, default_retval=True)( pair=trade.pair, trade=trade, order_type=order_type, amount=amount, rate=limit, time_in_force=time_in_force, sell_reason=sell_reason.sell_reason, - current_time=datetime.now(timezone.utc)): - logger.info(f"User requested abortion of selling {trade.pair}") + current_time=datetime.now(timezone.utc)): # TODO-lev: Update to exit + logger.info(f"User requested abortion of exiting {trade.pair}") return False try: # Execute sell and update trade record - order = self.exchange.create_order(pair=trade.pair, - ordertype=order_type, side="sell", - amount=amount, rate=limit, - time_in_force=time_in_force - ) + order = self.exchange.create_order( + pair=trade.pair, + ordertype=order_type, + side="sell", + amount=amount, + rate=limit, + time_in_force=time_in_force + ) except InsufficientFundsError as e: logger.warning(f"Unable to place order {e}.") # Try to figure out what went wrong @@ -1138,15 +1154,15 @@ class FreqtradeBot(LoggingMixin): trade.orders.append(order_obj) trade.open_order_id = order['id'] - trade.sell_order_status = '' + trade.sell_order_status = '' # TODO-lev: Update to exit_order_status trade.close_rate_requested = limit - trade.sell_reason = sell_reason.sell_reason + trade.sell_reason = sell_reason.sell_reason # TODO-lev: Update to exit_reason # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') in ('closed', 'expired'): self.update_trade_state(trade, trade.open_order_id, order) Trade.commit() - # Lock pair for one candle to prevent immediate re-buys + # Lock pair for one candle to prevent immediate re-trading self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), reason='Auto lock') @@ -1181,7 +1197,7 @@ class FreqtradeBot(LoggingMixin): 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_ratio': profit_ratio, - 'sell_reason': trade.sell_reason, + 'sell_reason': trade.sell_reason, # TODO-lev: change to exit_reason 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.utcnow(), 'stake_currency': self.config['stake_currency'], @@ -1200,10 +1216,10 @@ class FreqtradeBot(LoggingMixin): """ Sends rpc notification when a sell cancel occurred. """ - if trade.sell_order_status == reason: + if trade.sell_order_status == reason: # TODO-lev: Update to exit_order_status return else: - trade.sell_order_status = reason + trade.sell_order_status = reason # TODO-lev: Update to exit_order_status profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) @@ -1224,7 +1240,7 @@ class FreqtradeBot(LoggingMixin): 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_ratio': profit_ratio, - 'sell_reason': trade.sell_reason, + 'sell_reason': trade.sell_reason, # TODO-lev: trade to exit_reason 'open_date': trade.open_date, 'close_date': trade.close_date, 'stake_currency': self.config['stake_currency'], @@ -1310,6 +1326,7 @@ class FreqtradeBot(LoggingMixin): self.wallets.update() if fee_abs != 0 and self.wallets.get_free(trade_base_currency) >= amount: # Eat into dust if we own more than base currency + # TODO-lev: won't be in "base"(quote) currency for shorts logger.info(f"Fee amount for {trade} was in base currency - " f"Eating Fee {fee_abs} into dust.") elif fee_abs != 0: @@ -1386,6 +1403,7 @@ class FreqtradeBot(LoggingMixin): trade.update_fee(fee_cost, fee_currency, fee_rate, order.get('side', '')) if not isclose(amount, order_amount, abs_tol=constants.MATH_CLOSE_PREC): + # TODO-lev: leverage? logger.warning(f"Amount {amount} does not match amount {trade.amount}") raise DependencyException("Half bought? Amounts don't match") diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 99d4c60d0..084142646 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -386,7 +386,7 @@ class Backtesting: detail_data = detail_data.loc[ (detail_data['date'] >= sell_candle_time) & (detail_data['date'] < sell_candle_end) - ] + ] if len(detail_data) == 0: # Fall back to "regular" data if no detail data was found for this candle return self._get_sell_trade_entry_for_candle(trade, sell_row) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index b73611c1b..a57cf0821 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -549,7 +549,7 @@ class LocalTrade(): if self.is_open: payment = "BUY" if self.is_short else "SELL" # TODO-lev: On shorts, you buy a little bit more than the amount (amount + interest) - # This wll only print the original amount + # TODO-lev: This wll only print the original amount logger.info(f'{order_type.upper()}_{payment} has been fulfilled for {self}.') # TODO-lev: Double check this self.close(safe_value_fallback(order, 'average', 'price')) diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index face79729..93b5e90e2 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -127,7 +127,7 @@ class PairListManager(): :return: pairlist - whitelisted pairs """ try: - + # TODO-lev: filter for pairlists that are able to trade at the desired leverage whitelist = expand_pairlist(pairlist, self._exchange.get_markets().keys(), keep_invalid) except ValueError as err: logger.error(f"Pair whitelist contains an invalid Wildcard: {err}") diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 95a37452b..8c57237ec 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -36,6 +36,7 @@ class RPCException(Exception): raise RPCException('*Status:* `no active trade`') """ + # TODO-lev: Add new configuration options introduced with leveraged/short trading def __init__(self, message: str) -> None: super().__init__(self) diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 911d7d6c2..1250e7b92 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -90,7 +90,7 @@ def test_enter_exit_side(fee): @pytest.mark.usefixtures("init_persistence") -def test__set_stop_loss_isolated_liq(fee): +def test_set_stop_loss_isolated_liq(fee): trade = Trade( id=2, pair='ADA/USDT', From f5b01443adc143d090f120c6ca73e14437d13c5e Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Thu, 9 Sep 2021 02:10:12 -0600 Subject: [PATCH 37/58] buy/short -> entry order, sell/exit_short -> exit order --- freqtrade/freqtradebot.py | 10 +++++----- freqtrade/strategy/interface.py | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index ec2745b03..45a054ed6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -352,7 +352,7 @@ class FreqtradeBot(LoggingMixin): def enter_positions(self) -> int: """ - Tries to execute long buy/short sell orders for new trades (positions) + Tries to execute entry orders for new trades (positions) """ trades_created = 0 @@ -600,7 +600,7 @@ class FreqtradeBot(LoggingMixin): def _notify_buy(self, trade: Trade, order_type: str) -> None: """ - Sends rpc notification when a buy/short occurred. + Sends rpc notification when a entry order occurred. """ msg = { 'trade_id': trade.id, @@ -623,7 +623,7 @@ class FreqtradeBot(LoggingMixin): def _notify_buy_cancel(self, trade: Trade, order_type: str, reason: str) -> None: """ - Sends rpc notification when a buy/short cancel occurred. + Sends rpc notification when a entry order cancel occurred. """ current_rate = self.exchange.get_rate(trade.pair, refresh=False, side="buy") @@ -669,7 +669,7 @@ class FreqtradeBot(LoggingMixin): def exit_positions(self, trades: List[Any]) -> int: """ - Tries to execute sell/exit_short orders for open trades (positions) + Tries to execute exit orders for open trades (positions) """ trades_closed = 0 for trade in trades: @@ -1012,7 +1012,7 @@ class FreqtradeBot(LoggingMixin): def handle_cancel_sell(self, trade: Trade, order: Dict, reason: str) -> str: """ - Sell/exit_short cancel - cancel order and update trade + exit order cancel - cancel order and update trade :return: Reason for cancel """ # if trade is not partially completed, just cancel the order diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 194ea557a..4730e9fe1 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -168,7 +168,7 @@ class IStrategy(ABC, HyperStrategyMixin): """ Check buy enter timeout function callback. This method can be used to override the enter-timeout. - It is called whenever a limit buy/short order has been created, + It is called whenever a limit entry order has been created, and is not yet fully filled. Configuration options in `unfilledtimeout` will be verified before this, so ensure to set these timeouts high enough. @@ -178,7 +178,7 @@ class IStrategy(ABC, HyperStrategyMixin): :param trade: trade object. :param order: Order dictionary as returned from CCXT. :param **kwargs: Ensure to keep this here so updates to this won't break your strategy. - :return bool: When True is returned, then the buy/short-order is cancelled. + :return bool: When True is returned, then the entry order is cancelled. """ return False @@ -212,7 +212,7 @@ class IStrategy(ABC, HyperStrategyMixin): def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, time_in_force: str, current_time: datetime, **kwargs) -> bool: """ - Called right before placing a buy/short order. + Called right before placing a entry order. Timing for this function is critical, so avoid doing heavy computations or network requests in this method. @@ -236,7 +236,7 @@ class IStrategy(ABC, HyperStrategyMixin): rate: float, time_in_force: str, sell_reason: str, current_time: datetime, **kwargs) -> bool: """ - Called right before placing a regular sell/exit_short order. + Called right before placing a regular exit order. Timing for this function is critical, so avoid doing heavy computations or network requests in this method. @@ -410,7 +410,7 @@ class IStrategy(ABC, HyperStrategyMixin): Checks if a pair is currently locked The 2nd, optional parameter ensures that locks are applied until the new candle arrives, and not stop at 14:00:00 - while the next candle arrives at 14:00:02 leaving a gap - of 2 seconds for a buy/short to happen on an old signal. + of 2 seconds for an entry order to happen on an old signal. :param pair: "Pair to check" :param candle_date: Date of the last candle. Optional, defaults to current date :returns: locking state of the pair in question. @@ -426,7 +426,7 @@ class IStrategy(ABC, HyperStrategyMixin): def analyze_ticker(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ Parses the given candle (OHLCV) data and returns a populated DataFrame - add several TA indicators and buy/short signal to it + add several TA indicators and entry order signal to it :param dataframe: Dataframe containing data from exchange :param metadata: Metadata dictionary with additional data (e.g. 'pair') :return: DataFrame of candle (OHLCV) data with indicator data and signals added @@ -541,7 +541,7 @@ class IStrategy(ABC, HyperStrategyMixin): dataframe: DataFrame ) -> Tuple[bool, bool, Optional[str]]: """ - Calculates current signal based based on the buy/short or sell/exit_short + Calculates current signal based based on the entry order or exit order columns of the dataframe. Used by Bot to get the signal to buy, sell, short, or exit_short :param pair: pair in format ANT/BTC @@ -606,7 +606,7 @@ class IStrategy(ABC, HyperStrategyMixin): sell: bool, low: float = None, high: float = None, force_stoploss: float = 0) -> SellCheckTuple: """ - This function evaluates if one of the conditions required to trigger a sell/exit_short + This function evaluates if one of the conditions required to trigger an exit order has been reached, which can either be a stop-loss, ROI or exit-signal. :param low: Only used during backtesting to simulate (long)stoploss/(short)ROI :param high: Only used during backtesting, to simulate (short)stoploss/(long)ROI @@ -810,7 +810,7 @@ class IStrategy(ABC, HyperStrategyMixin): def advise_buy(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators, populates the buy/short signal for the given dataframe + Based on TA indicators, populates the entry order signal for the given dataframe This method should not be overridden. :param dataframe: DataFrame :param metadata: Additional information dictionary, with details like the @@ -829,7 +829,7 @@ class IStrategy(ABC, HyperStrategyMixin): def advise_sell(self, dataframe: DataFrame, metadata: dict) -> DataFrame: """ - Based on TA indicators, populates the sell/exit_short signal for the given dataframe + Based on TA indicators, populates the exit order signal for the given dataframe This method should not be overridden. :param dataframe: DataFrame :param metadata: Additional information dictionary, with details like the From 366247dff3bdd360b5b5fdc43c36e1396c902d91 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Thu, 9 Sep 2021 02:16:24 -0600 Subject: [PATCH 38/58] removed caplog.clears at end of functions in test_freqtradebot --- tests/test_freqtradebot.py | 45 -------------------------------------- 1 file changed, 45 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index ddd031f77..a9760978e 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -107,7 +107,6 @@ def test_order_dict_dry_run(default_conf, mocker, caplog) -> None: freqtrade = FreqtradeBot(conf) assert not freqtrade.strategy.order_types['stoploss_on_exchange'] assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) - caplog.clear() def test_order_dict_live(default_conf, mocker, caplog) -> None: @@ -141,7 +140,6 @@ def test_order_dict_live(default_conf, mocker, caplog) -> None: freqtrade = FreqtradeBot(conf) assert not freqtrade.strategy.order_types['stoploss_on_exchange'] assert not log_has_re(".*stoploss_on_exchange .* dry-run", caplog) - caplog.clear() def test_get_trade_stake_amount(default_conf, ticker, mocker) -> None: @@ -417,7 +415,6 @@ def test_create_trade_too_small_stake_amount(default_conf, ticker, limit_buy_ord assert freqtrade.create_trade('ETH/BTC') assert log_has_re(r"Stake amount for pair .* is too small.*", caplog) - caplog.clear() def test_create_trade_zero_stake_amount(default_conf, ticker, limit_buy_order_open, @@ -481,7 +478,6 @@ def test_enter_positions_no_pairs_left(default_conf, ticker, limit_buy_order_ope n = freqtrade.enter_positions() assert n == 0 assert log_has_re(r"No currency pair in active pair whitelist.*", caplog) - caplog.clear() def test_enter_positions_no_pairs_in_whitelist(default_conf, ticker, limit_buy_order, fee, @@ -528,7 +524,6 @@ def test_enter_positions_global_pairlock(default_conf, ticker, limit_buy_order, n = freqtrade.enter_positions() assert n == 0 assert log_has_re(message, caplog) - caplog.clear() def test_create_trade_no_signal(default_conf, fee, mocker) -> None: @@ -1122,7 +1117,6 @@ def test_handle_stoploss_on_exchange(mocker, default_conf, fee, caplog, mocker.patch('freqtrade.exchange.Binance.stoploss', stoploss) assert freqtrade.handle_stoploss_on_exchange(trade) is False assert stoploss.call_count == 0 - caplog.clear() def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, @@ -1162,7 +1156,6 @@ def test_handle_sle_cancel_cant_recreate(mocker, default_conf, fee, caplog, assert log_has_re(r'Stoploss order was cancelled, but unable to recreate one.*', caplog) assert trade.stoploss_order_id is None assert trade.is_open is True - caplog.clear() def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, @@ -1211,7 +1204,6 @@ def test_create_stoploss_order_invalid_order(mocker, default_conf, caplog, fee, assert rpc_mock.call_count == 2 assert rpc_mock.call_args_list[1][0][0]['sell_reason'] == SellType.EMERGENCY_SELL.value assert rpc_mock.call_args_list[1][0][0]['order_type'] == 'market' - caplog.clear() def test_create_stoploss_order_insufficient_funds(mocker, default_conf, caplog, fee, @@ -1438,7 +1430,6 @@ def test_handle_stoploss_on_exchange_trailing_error(mocker, default_conf, fee, c freqtrade.handle_trailing_stoploss_on_exchange(trade, stoploss_order_hanging) assert cancel_mock.call_count == 1 assert log_has_re(r"Could not create trailing stoploss order for pair ETH/BTC\..*", caplog) - caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -1673,7 +1664,6 @@ def test_enter_positions(mocker, default_conf, caplog) -> None: assert log_has('Found no buy signals for whitelisted currencies. Trying again...', caplog) # create_trade should be called once for every pair in the whitelist. assert mock_ct.call_count == len(default_conf['exchange']['pair_whitelist']) - caplog.clear() def test_enter_positions_exception(mocker, default_conf, caplog) -> None: @@ -1713,7 +1703,6 @@ def test_exit_positions(mocker, default_conf, limit_buy_order, caplog) -> None: # test amount modified by fee-logic n = freqtrade.exit_positions(trades) assert n == 0 - caplog.clear() def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1734,7 +1723,6 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) n = freqtrade.exit_positions(trades) assert n == 0 assert log_has('Unable to sell trade ETH/BTC: ', caplog) - caplog.clear() def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> None: @@ -1780,7 +1768,6 @@ def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> No freqtrade.update_trade_state(trade, '123') assert log_has_re('Found open order for.*', caplog) - caplog.clear() def test_update_trade_state_withorderdict(default_conf, trades_for_order, limit_buy_order, fee, @@ -1831,7 +1818,6 @@ def test_update_trade_state_withorderdict_rounding_fee(default_conf, trades_for_ assert trade.amount != amount assert trade.amount == limit_buy_order['amount'] assert log_has_re(r'Applying fee on amount for .*', caplog) - caplog.clear() def test_update_trade_state_exception(mocker, default_conf, @@ -1850,7 +1836,6 @@ def test_update_trade_state_exception(mocker, default_conf, ) freqtrade.update_trade_state(trade, trade.open_order_id) assert log_has('Could not update trade amount: ', caplog) - caplog.clear() def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None: @@ -1867,7 +1852,6 @@ def test_update_trade_state_orderexception(mocker, default_conf, caplog) -> None freqtrade.update_trade_state(trade, trade.open_order_id) assert grm_mock.call_count == 0 assert log_has(f'Unable to fetch order {trade.open_order_id}: ', caplog) - caplog.clear() def test_update_trade_state_sell(default_conf, trades_for_order, limit_sell_order_open, @@ -2036,7 +2020,6 @@ def test_handle_trade_roi(default_conf, ticker, limit_buy_order_open, assert freqtrade.handle_trade(trade) assert log_has("ETH/BTC - Required profit reached. sell_type=SellType.ROI", caplog) - caplog.clear() def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open, @@ -2069,7 +2052,6 @@ def test_handle_trade_use_sell_signal(default_conf, ticker, limit_buy_order_open assert freqtrade.handle_trade(trade) assert log_has("ETH/BTC - Sell signal received. sell_type=SellType.SELL_SIGNAL", caplog) - caplog.clear() def test_close_trade(default_conf, ticker, limit_buy_order, limit_buy_order_open, limit_sell_order, @@ -2110,7 +2092,6 @@ def test_bot_loop_start_called_once(mocker, default_conf, caplog): assert log_has_re(r'Strategy caused the following exception.*', caplog) assert ftbot.strategy.bot_loop_start.call_count == 1 assert ftbot.strategy.analyze.call_count == 1 - caplog.clear() def test_check_handle_timedout_buy_usercustom(default_conf, ticker, limit_buy_order_old, open_trade, @@ -2225,7 +2206,6 @@ def test_check_handle_cancelled_buy(default_conf, ticker, limit_buy_order_old, o nb_trades = len(trades) assert nb_trades == 0 assert log_has_re("Buy order cancelled on exchange for Trade.*", caplog) - caplog.clear() def test_check_handle_timedout_buy_exception(default_conf, ticker, limit_buy_order_old, open_trade, @@ -2360,7 +2340,6 @@ def test_check_handle_cancelled_sell(default_conf, ticker, limit_sell_order_old, assert rpc_mock.call_count == 1 assert open_trade.is_open is True assert log_has_re("Sell order cancelled on exchange for Trade.*", caplog) - caplog.clear() def test_check_handle_timedout_partial(default_conf, ticker, limit_buy_order_old_partial, @@ -2429,7 +2408,6 @@ def test_check_handle_timedout_partial_fee(default_conf, ticker, open_trade, cap assert trades[0].open_order_id is None assert trades[0].fee_updated('buy') assert pytest.approx(trades[0].fee_open) == 0.001 - caplog.clear() def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, caplog, fee, @@ -2470,7 +2448,6 @@ def test_check_handle_timedout_partial_except(default_conf, ticker, open_trade, limit_buy_order_old_partial['remaining']) assert trades[0].open_order_id is None assert trades[0].fee_open == fee() - caplog.clear() def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocker, caplog) -> None: @@ -2499,7 +2476,6 @@ def test_check_handle_timedout_exception(default_conf, ticker, open_trade, mocke f"{open_trade.open_date.strftime('%Y-%m-%d %H:%M:%S')}" r"\) due to Traceback \(most recent call last\):\n*", caplog) - caplog.clear() def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> None: @@ -2543,7 +2519,6 @@ def test_handle_cancel_buy(mocker, caplog, default_conf, limit_buy_order) -> Non mocker.patch('freqtrade.exchange.Exchange.cancel_order_with_result', cancel_order_mock) assert not freqtrade.handle_cancel_buy(trade, limit_buy_order, reason) assert log_has_re(r"Order .* for .* not cancelled.", caplog) - caplog.clear() @pytest.mark.parametrize("limit_buy_order_canceled_empty", ['binance', 'ftx', 'kraken', 'bittrex'], @@ -2565,7 +2540,6 @@ def test_handle_cancel_buy_exchanges(mocker, caplog, default_conf, assert cancel_order_mock.call_count == 0 assert log_has_re(r'Buy order fully cancelled. Removing .* from database\.', caplog) assert nofiy_mock.call_count == 1 - caplog.clear() @pytest.mark.parametrize('cancelorder', [ @@ -2935,7 +2909,6 @@ def test_execute_trade_exit_sloe_cancel_exception( sell_reason=SellCheckTuple(sell_type=SellType.STOP_LOSS)) assert create_order_mock.call_count == 2 assert log_has('Could not cancel stoploss order abcd', caplog) - caplog.clear() def test_execute_trade_exit_with_stoploss_on_exchange(default_conf, ticker, fee, ticker_sell_up, @@ -3331,7 +3304,6 @@ def test_sell_not_enough_balance(default_conf, limit_buy_order, limit_buy_order_ assert freqtrade.handle_trade(trade) is True assert log_has_re(r'.*Falling back to wallet-amount.', caplog) assert trade.amount != amnt - caplog.clear() def test__safe_sell_amount(default_conf, fee, caplog, mocker): @@ -3362,7 +3334,6 @@ def test__safe_sell_amount(default_conf, fee, caplog, mocker): assert freqtrade._safe_sell_amount(trade.pair, amount_wallet) == amount_wallet assert not log_has_re(r'.*Falling back to wallet-amount.', caplog) assert wallet_update.call_count == 1 - caplog.clear() def test__safe_sell_amount_error(default_conf, fee, caplog, mocker): @@ -3419,7 +3390,6 @@ def test_locked_pairs(default_conf, ticker, fee, ticker_sell_down, mocker, caplo freqtrade.enter_positions() assert log_has_re(f"Pair {trade.pair} is still locked.*", caplog) - caplog.clear() def test_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, @@ -3511,7 +3481,6 @@ def test_trailing_stop_loss(default_conf, limit_buy_order_open, limit_buy_order, assert log_has("ETH/BTC - HIT STOP: current price at 0.000012, stoploss is 0.000015, " "initial stoploss was at 0.000010, trade opened at 0.000011", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value - caplog.clear() def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3573,7 +3542,6 @@ def test_trailing_stop_loss_positive(default_conf, limit_buy_order, limit_buy_or f"ETH/BTC - HIT STOP: current price at {buy_price + 0.000002:.6f}, " f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) - caplog.clear() def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3636,7 +3604,6 @@ def test_trailing_stop_loss_offset(default_conf, limit_buy_order, limit_buy_orde f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at 0.000010, trade opened at 0.000011", caplog) assert trade.sell_reason == SellType.TRAILING_STOP_LOSS.value - caplog.clear() def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_open, fee, @@ -3701,7 +3668,6 @@ def test_tsl_only_offset_reached(default_conf, limit_buy_order, limit_buy_order_ assert log_has("ETH/BTC - Using positive stoploss: 0.05 offset: 0.055 profit: 0.1218%", caplog) assert log_has("ETH/BTC - Adjusting stoploss...", caplog) assert trade.stop_loss == 0.0000117705 - caplog.clear() def test_disable_ignore_roi_if_buy_signal(default_conf, limit_buy_order, limit_buy_order_open, @@ -3763,7 +3729,6 @@ def test_get_real_amount_quote(default_conf, trades_for_order, buy_order_fee, fe assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.992).', caplog) - caplog.clear() def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fee, fee, @@ -3789,7 +3754,6 @@ def test_get_real_amount_quote_dust(default_conf, trades_for_order, buy_order_fe assert walletmock.call_count == 1 assert log_has_re(r'Fee amount for Trade.* was in base currency ' '- Eating Fee 0.008 into dust', caplog) - caplog.clear() def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, fee): @@ -3812,7 +3776,6 @@ def test_get_real_amount_no_trade(default_conf, buy_order_fee, caplog, mocker, f assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) failed: myTrade-Dict empty found', caplog) - caplog.clear() def test_get_real_amount_stake(default_conf, trades_for_order, buy_order_fee, fee, mocker): @@ -3906,7 +3869,6 @@ def test_get_real_amount_multi(default_conf, trades_for_order2, buy_order_fee, c assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None - caplog.clear() def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, caplog, fee, @@ -3942,7 +3904,6 @@ def test_get_real_amount_multi2(default_conf, trades_for_order3, buy_order_fee, assert trade.fee_open_currency is not None assert trade.fee_close_cost is None assert trade.fee_close_currency is None - caplog.clear() def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee, fee, @@ -3971,7 +3932,6 @@ def test_get_real_amount_fromorder(default_conf, trades_for_order, buy_order_fee assert log_has('Applying fee on amount for Trade(id=None, pair=LTC/ETH, amount=8.00000000, ' 'open_rate=0.24544100, open_since=closed) (from 8.0 to 7.996).', caplog) - caplog.clear() def test_get_real_amount_invalid_order(default_conf, trades_for_order, buy_order_fee, fee, mocker): @@ -4215,7 +4175,6 @@ def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None with pytest.raises(PricingError): freqtrade.exchange.get_rate('ETH/BTC', refresh=True, side="buy") assert log_has_re(r'Buy Price at location 1 from orderbook could not be determined.', caplog) - caplog.clear() def test_check_depth_of_market_buy(default_conf, mocker, order_book_l2) -> None: @@ -4286,7 +4245,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o freqtrade.handle_trade(trade) assert log_has_re(r'Sell Price at location 1 from orderbook could not be determined\..*', caplog) - caplog.clear() def test_startup_state(default_conf, mocker): @@ -4345,7 +4303,6 @@ def test_sync_wallet_dry_run(mocker, default_conf, ticker, fee, limit_buy_order_ assert log_has_re(r"Unable to create trade for XRP/BTC: " r"Available balance \(0.0 BTC\) is lower than stake amount \(0.001 BTC\)", caplog) - caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -4505,7 +4462,6 @@ def test_reupdate_buy_order_fees(mocker, default_conf, fee, caplog): assert log_has_re(r"Trying to reupdate buy fees for .*", caplog) assert mock_uts.call_count == 0 assert not log_has_re(r"Updating buy-fee on trade .* for order .*\.", caplog) - caplog.clear() @pytest.mark.usefixtures("init_persistence") @@ -4643,7 +4599,6 @@ def test_refind_lost_order(mocker, default_conf, fee, caplog): freqtrade.refind_lost_order(trades[4]) assert log_has(f"Error updating {order['id']}.", caplog) - caplog.clear() def test_get_valid_price(mocker, default_conf) -> None: From ee874f461c19f0365f41503f25e58c0fd64ce9e7 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Thu, 9 Sep 2021 13:14:48 -0600 Subject: [PATCH 39/58] Removed TODO: change to exit-reason, exit_order_status --- freqtrade/freqtradebot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 45a054ed6..fe7261089 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -1154,9 +1154,9 @@ class FreqtradeBot(LoggingMixin): trade.orders.append(order_obj) trade.open_order_id = order['id'] - trade.sell_order_status = '' # TODO-lev: Update to exit_order_status + trade.sell_order_status = '' trade.close_rate_requested = limit - trade.sell_reason = sell_reason.sell_reason # TODO-lev: Update to exit_reason + trade.sell_reason = sell_reason.sell_reason # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') in ('closed', 'expired'): self.update_trade_state(trade, trade.open_order_id, order) @@ -1197,7 +1197,7 @@ class FreqtradeBot(LoggingMixin): 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_ratio': profit_ratio, - 'sell_reason': trade.sell_reason, # TODO-lev: change to exit_reason + 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, 'close_date': trade.close_date or datetime.utcnow(), 'stake_currency': self.config['stake_currency'], @@ -1216,10 +1216,10 @@ class FreqtradeBot(LoggingMixin): """ Sends rpc notification when a sell cancel occurred. """ - if trade.sell_order_status == reason: # TODO-lev: Update to exit_order_status + if trade.sell_order_status == reason: return else: - trade.sell_order_status = reason # TODO-lev: Update to exit_order_status + trade.sell_order_status = reason profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) @@ -1240,7 +1240,7 @@ class FreqtradeBot(LoggingMixin): 'current_rate': current_rate, 'profit_amount': profit_trade, 'profit_ratio': profit_ratio, - 'sell_reason': trade.sell_reason, # TODO-lev: trade to exit_reason + 'sell_reason': trade.sell_reason, 'open_date': trade.open_date, 'close_date': trade.close_date, 'stake_currency': self.config['stake_currency'], From e1a749a91e53d1a7abcbc9ab30adb606bd773924 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Thu, 9 Sep 2021 13:19:43 -0600 Subject: [PATCH 40/58] removed unnecessary caplog --- tests/test_freqtradebot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index fc4b6fb74..180848c9c 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -1721,7 +1721,6 @@ def test_exit_positions_exception(mocker, default_conf, limit_buy_order, caplog) n = freqtrade.exit_positions(trades) assert n == 0 assert log_has('Unable to exit trade ETH/BTC: ', caplog) - caplog.clear() def test_update_trade_state(mocker, default_conf, limit_buy_order, caplog) -> None: From 982534ddc70b7db43c598d7a48a2ff63c81aaedf Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Sep 2021 20:56:46 +0200 Subject: [PATCH 41/58] Add gate.io to list of supported exchanges --- README.md | 1 + docs/index.md | 1 + freqtrade/exchange/gateio.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 309fab94b..2c164135f 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Please read the [exchange specific notes](docs/exchanges.md) to learn about even - [X] [Bittrex](https://bittrex.com/) - [X] [Kraken](https://kraken.com/) - [X] [FTX](https://ftx.com) +- [X] [Gate.io](https://www.gate.io/ref/6266643) - [ ] [potentially many others](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ### Community tested diff --git a/docs/index.md b/docs/index.md index fd3b8f224..7735117e2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -40,6 +40,7 @@ Please read the [exchange specific notes](exchanges.md) to learn about eventual, - [X] [Bittrex](https://bittrex.com/) - [X] [FTX](https://ftx.com) - [X] [Kraken](https://kraken.com/) +- [X] [Gate.io](https://www.gate.io/ref/6266643) - [ ] [potentially many others through ccxt](https://github.com/ccxt/ccxt/). _(We cannot guarantee they will work)_ ### Community tested diff --git a/freqtrade/exchange/gateio.py b/freqtrade/exchange/gateio.py index 9c910a10d..e6ee01c8a 100644 --- a/freqtrade/exchange/gateio.py +++ b/freqtrade/exchange/gateio.py @@ -21,3 +21,5 @@ class Gateio(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, } + + _headers = {'X-Gate-Channel-Id': 'freqtrade'} From a19c33ba54ff051dfa20aae864510ac2a8bb6626 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Sep 2021 05:52:48 +0000 Subject: [PATCH 42/58] Don't blindly create coroutines, but fire them off in batches --- freqtrade/exchange/exchange.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 79fd33dfe..48c3caa50 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -28,7 +28,7 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, retrier_async) -from freqtrade.misc import deep_merge_dicts, safe_value_fallback2 +from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -1238,19 +1238,20 @@ class Exchange: input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - - results = await asyncio.gather(*input_coroutines, return_exceptions=True) - # Combine gathered results + data: List = [] - for res in results: - if isinstance(res, Exception): - logger.warning("Async code raised an exception: %s", res.__class__.__name__) - continue - # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: - data.extend(new_data) + for input_coro in chunks(input_coroutines, 100): + + results = await asyncio.gather(*input_coro, return_exceptions=True) + for res in results: + if isinstance(res, Exception): + logger.warning("Async code raised an exception: %s", res.__class__.__name__) + continue + # Deconstruct tuple if it's not an exception + p, _, new_data = res + if p == pair: + data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) logger.info("Downloaded data for %s with length %s.", pair, len(data)) From 8c9159f5969f593aa36754af9f19e4667223ba0a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Sep 2021 19:46:38 +0200 Subject: [PATCH 43/58] Improve comments --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 48c3caa50..e79d53e80 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1238,9 +1238,9 @@ class Exchange: input_coroutines = [self._async_get_candle_history( pair, timeframe, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] - # Combine gathered results data: List = [] + # Chunk requests into batches of 100 to avoid overwelming ccxt Throttling for input_coro in chunks(input_coroutines, 100): results = await asyncio.gather(*input_coro, return_exceptions=True) From 4c4604f837d05d1d3c391031f24e8b63cef2da48 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Sep 2021 19:47:56 +0200 Subject: [PATCH 44/58] Add explicit test for get_historic_ohlcv --- tests/exchange/test_exchange.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 144063c07..45c2e627c 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1551,6 +1551,32 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): assert 'high' in ret.columns +@pytest.mark.asyncio +@pytest.mark.parametrize("exchange_name", EXCHANGES) +async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): + ohlcv = [ + [ + int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), + 1, # open + 2, # high + 3, # low + 4, # close + 5, # volume (in quote currency) + ] + ] + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + # Monkey-patch async function + exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) + + pair = 'ETH/BTC' + res = await exchange._async_get_historic_ohlcv(pair, "5m", + 1500000000000, is_new_pair=False) + # Call with very old timestamp - causes tons of requests + assert exchange._api_async.fetch_ohlcv.call_count > 200 + assert res[0] == ohlcv[0] + assert log_has_re(r'Downloaded data for .* with length .*\.', caplog) + + def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: ohlcv = [ [ From 84c121652acde5e50f8989df3889adf2146960da Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 10 Sep 2021 23:42:16 -0600 Subject: [PATCH 45/58] Added more todos --- freqtrade/commands/hyperopt_commands.py | 1 + freqtrade/commands/list_commands.py | 1 + freqtrade/plugins/pairlist/PrecisionFilter.py | 1 + freqtrade/plugins/protections/max_drawdown_protection.py | 1 + freqtrade/plugins/protections/stoploss_guard.py | 2 ++ 5 files changed, 6 insertions(+) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index 089529d15..d2d30f399 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -102,3 +102,4 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, header_str="Epoch details") +# TODO-lev: Hyperopt optimal leverage diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 410b9b72b..2b857cba6 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -148,6 +148,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: quote_currencies = args.get('quote_currencies', []) try: + # TODO-lev: Add leverage amount to get markets that support a certain leverage pairs = exchange.get_markets(base_currencies=base_currencies, quote_currencies=quote_currencies, pairs_only=pairs_only, diff --git a/freqtrade/plugins/pairlist/PrecisionFilter.py b/freqtrade/plugins/pairlist/PrecisionFilter.py index a3c262e8c..2c02ccdb3 100644 --- a/freqtrade/plugins/pairlist/PrecisionFilter.py +++ b/freqtrade/plugins/pairlist/PrecisionFilter.py @@ -18,6 +18,7 @@ class PrecisionFilter(IPairList): pairlist_pos: int) -> None: super().__init__(exchange, pairlistmanager, config, pairlistconfig, pairlist_pos) + # TODO-lev: Liquidation price? if 'stoploss' not in self._config: raise OperationalException( 'PrecisionFilter can only work with stoploss defined. Please add the ' diff --git a/freqtrade/plugins/protections/max_drawdown_protection.py b/freqtrade/plugins/protections/max_drawdown_protection.py index 67e204039..89b723c60 100644 --- a/freqtrade/plugins/protections/max_drawdown_protection.py +++ b/freqtrade/plugins/protections/max_drawdown_protection.py @@ -36,6 +36,7 @@ class MaxDrawdown(IProtection): """ LockReason to use """ + # TODO-lev: < for shorts? return (f'{drawdown} > {self._max_allowed_drawdown} in {self.lookback_period_str}, ' f'locking for {self.stop_duration_str}.') diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 40edf1204..888dc0316 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -32,6 +32,7 @@ class StoplossGuard(IProtection): def _reason(self) -> str: """ LockReason to use + #TODO-lev: check if min is the right word for shorts """ return (f'{self._trade_limit} stoplosses in {self._lookback_period} min, ' f'locking for {self._stop_duration} min.') @@ -51,6 +52,7 @@ class StoplossGuard(IProtection): # if pair: # filters.append(Trade.pair == pair) # trades = Trade.get_trades(filters).all() + # TODO-lev: Liquidation price? trades1 = Trade.get_trades_proxy(pair=pair, is_open=False, close_date=look_back_until) trades = [trade for trade in trades1 if (str(trade.sell_reason) in ( From a5f90a409cb5c78373bf2c477ac5dc62958cbf70 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 08:18:32 +0200 Subject: [PATCH 46/58] Small updates to async_history_fetch --- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e79d53e80..11de5411f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1254,7 +1254,7 @@ class Exchange: data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - logger.info("Downloaded data for %s with length %s.", pair, len(data)) + logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 45c2e627c..90e19782e 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1568,7 +1568,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ # Monkey-patch async function exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) - pair = 'ETH/BTC' + pair = 'ETH/USDT' res = await exchange._async_get_historic_ohlcv(pair, "5m", 1500000000000, is_new_pair=False) # Call with very old timestamp - causes tons of requests From 432c3df17e7672ab33554d161a4f7fb3e1f218ac Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Sep 2021 08:04:22 +0200 Subject: [PATCH 47/58] Add documentation for Bittex/Gemini with VolumePairlist closes #5565 --- docs/exchanges.md | 6 ++++++ docs/includes/pairlists.md | 20 ++++++++++++++++++++ freqtrade/plugins/pairlist/VolumePairList.py | 2 +- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 42a850acd..c0fbdc694 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -58,6 +58,12 @@ Bittrex does not support market orders. If you have a message at the bot startup Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment. Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected. +### Volume pairlist + +Bittrex does not support the direct usage of VolumePairList. This can however be worked around by using the advanced mode with `lookback_days: 1` (or more), which will emulate 24h volume. + +Read more in the [pairlist documentation](plugins.md#volumepairlist-advanced-mode). + ### Restricted markets Bittrex split its exchange into US and International versions. diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index 6e23c9003..69e12d5dc 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -82,6 +82,8 @@ Filtering instances (not the first position in the list) will not apply any cach You can define a minimum volume with `min_value` - which will filter out pairs with a volume lower than the specified value in the specified timerange. +### VolumePairList Advanced mode + `VolumePairList` can also operate in an advanced mode to build volume over a given timerange of specified candle size. It utilizes exchange historical candle data, builds a typical price (calculated by (open+high+low)/3) and multiplies the typical price with every candle's volume. The sum is the `quoteVolume` over the given range. This allows different scenarios, for a more smoothened volume, when using longer ranges with larger candle sizes, or the opposite when using a short range with small candles. For convenience `lookback_days` can be specified, which will imply that 1d candles will be used for the lookback. In the example below the pairlist would be created based on the last 7 days: @@ -105,6 +107,24 @@ For convenience `lookback_days` can be specified, which will imply that 1d candl !!! Warning "Performance implications when using lookback range" If used in first position in combination with lookback, the computation of the range based volume can be time and resource consuming, as it downloads candles for all tradable pairs. Hence it's highly advised to use the standard approach with `VolumeFilter` to narrow the pairlist down for further range volume calculation. +??? Tip "Unsupported exchanges (Bittrex, Gemini)" + On some exchanges (like Bittrex and Gemini), regular VolumePairList does not work as the api does not natively provide 24h volume. This can be worked around by using candle data to build the volume. + To roughly simulate 24h volume, you can use the following configuration. + Please note that These pairlists will only refresh once per day. + + ```json + "pairlists": [ + { + "method": "VolumePairList", + "number_assets": 20, + "sort_key": "quoteVolume", + "min_value": 0, + "refresh_period": 86400, + "lookback_days": 1 + } + ], + ``` + More sophisticated approach can be used, by using `lookback_timeframe` for candle size and `lookback_period` which specifies the amount of candles. This example will build the volume pairs based on a rolling period of 3 days of 1h candles: ```json diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index c70e4a904..0ffc8a8c8 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -123,7 +123,7 @@ class VolumePairList(IPairList): filtered_tickers = [ v for k, v in tickers.items() if (self._exchange.get_pair_quote_currency(k) == self._stake_currency - and v[self._sort_key] is not None)] + and (self._use_range or v[self._sort_key] is not None))] pairlist = [s['symbol'] for s in filtered_tickers] pairlist = self.filter_pairlist(pairlist, tickers) From ebb0b8aa3fd03100d96e8feef4a4f9919fab60fe Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 08:36:57 +0200 Subject: [PATCH 48/58] Remove new-hyperopt command --- docs/utils.md | 35 ----- freqtrade/commands/__init__.py | 2 +- freqtrade/commands/arguments.py | 17 +-- freqtrade/commands/deploy_commands.py | 52 +------ freqtrade/templates/base_hyperopt.py.j2 | 137 ------------------ .../subtemplates/hyperopt_buy_guards_full.j2 | 8 - .../hyperopt_buy_guards_minimal.j2 | 2 - .../subtemplates/hyperopt_buy_space_full.j2 | 9 -- .../hyperopt_buy_space_minimal.j2 | 3 - .../subtemplates/hyperopt_sell_guards_full.j2 | 8 - .../hyperopt_sell_guards_minimal.j2 | 2 - .../subtemplates/hyperopt_sell_space_full.j2 | 11 -- .../hyperopt_sell_space_minimal.j2 | 5 - tests/commands/test_commands.py | 33 +---- 14 files changed, 7 insertions(+), 317 deletions(-) delete mode 100644 freqtrade/templates/base_hyperopt.py.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 delete mode 100644 freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 diff --git a/docs/utils.md b/docs/utils.md index 6395fb6f9..a1e219d6a 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -111,41 +111,6 @@ Using the advanced template (populates all optional functions and methods) freqtrade new-strategy --strategy AwesomeStrategy --template advanced ``` -## Create new hyperopt - -Creates a new hyperopt from a template similar to SampleHyperopt. -The file will be named inline with your class name, and will not overwrite existing files. - -Results will be located in `user_data/hyperopts/.py`. - -``` output -usage: freqtrade new-hyperopt [-h] [--userdir PATH] [--hyperopt NAME] - [--template {full,minimal,advanced}] - -optional arguments: - -h, --help show this help message and exit - --userdir PATH, --user-data-dir PATH - Path to userdata directory. - --hyperopt NAME Specify hyperopt class name which will be used by the - bot. - --template {full,minimal,advanced} - Use a template which is either `minimal`, `full` - (containing multiple sample indicators) or `advanced`. - Default: `full`. -``` - -### Sample usage of new-hyperopt - -```bash -freqtrade new-hyperopt --hyperopt AwesomeHyperopt -``` - -With custom user directory - -```bash -freqtrade new-hyperopt --userdir ~/.freqtrade/ --hyperopt AwesomeHyperopt -``` - ## List Strategies and List Hyperopts Use the `list-strategies` subcommand to see all strategies in one particular directory and the `list-hyperopts` subcommand to list custom Hyperopts. diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 04e46ee23..49b184a80 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -11,7 +11,7 @@ from freqtrade.commands.build_config_commands import start_new_config from freqtrade.commands.data_commands import (start_convert_data, start_download_data, start_list_data) from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, - start_new_hyperopt, start_new_strategy) + start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 899998310..9c6ce543f 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -55,8 +55,6 @@ ARGS_BUILD_CONFIG = ["config"] ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] -ARGS_BUILD_HYPEROPT = ["user_data_dir", "hyperopt", "template"] - ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] @@ -95,7 +93,7 @@ NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list "list-hyperopts", "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit", "show-trades"] -NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-hyperopt", "new-strategy"] +NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] class Arguments: @@ -176,10 +174,9 @@ class Arguments: start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, - start_list_timeframes, start_new_config, start_new_hyperopt, - start_new_strategy, start_plot_dataframe, start_plot_profit, - start_show_trades, start_test_pairlist, start_trading, - start_webserver) + start_list_timeframes, start_new_config, start_new_strategy, + start_plot_dataframe, start_plot_profit, start_show_trades, + start_test_pairlist, start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -206,12 +203,6 @@ class Arguments: build_config_cmd.set_defaults(func=start_new_config) self._build_args(optionlist=ARGS_BUILD_CONFIG, parser=build_config_cmd) - # add new-hyperopt subcommand - build_hyperopt_cmd = subparsers.add_parser('new-hyperopt', - help="Create new hyperopt") - build_hyperopt_cmd.set_defaults(func=start_new_hyperopt) - self._build_args(optionlist=ARGS_BUILD_HYPEROPT, parser=build_hyperopt_cmd) - # add new-strategy subcommand build_strategy_cmd = subparsers.add_parser('new-strategy', help="Create new strategy") diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index c98335e0b..4f9e5bbad 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -7,7 +7,7 @@ import requests from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template, render_template_with_fallback @@ -87,56 +87,6 @@ def start_new_strategy(args: Dict[str, Any]) -> None: raise OperationalException("`new-strategy` requires --strategy to be set.") -def deploy_new_hyperopt(hyperopt_name: str, hyperopt_path: Path, subtemplate: str) -> None: - """ - Deploys a new hyperopt template to hyperopt_path - """ - fallback = 'full' - buy_guards = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_buy_guards_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_buy_guards_{fallback}.j2", - ) - sell_guards = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_sell_guards_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_sell_guards_{fallback}.j2", - ) - buy_space = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_buy_space_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_buy_space_{fallback}.j2", - ) - sell_space = render_template_with_fallback( - templatefile=f"subtemplates/hyperopt_sell_space_{subtemplate}.j2", - templatefallbackfile=f"subtemplates/hyperopt_sell_space_{fallback}.j2", - ) - - strategy_text = render_template(templatefile='base_hyperopt.py.j2', - arguments={"hyperopt": hyperopt_name, - "buy_guards": buy_guards, - "sell_guards": sell_guards, - "buy_space": buy_space, - "sell_space": sell_space, - }) - - logger.info(f"Writing hyperopt to `{hyperopt_path}`.") - hyperopt_path.write_text(strategy_text) - - -def start_new_hyperopt(args: Dict[str, Any]) -> None: - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - if 'hyperopt' in args and args['hyperopt']: - - new_path = config['user_data_dir'] / USERPATH_HYPEROPTS / (args['hyperopt'] + '.py') - - if new_path.exists(): - raise OperationalException(f"`{new_path}` already exists. " - "Please choose another Hyperopt Name.") - deploy_new_hyperopt(args['hyperopt'], new_path, args['template']) - else: - raise OperationalException("`new-hyperopt` requires --hyperopt to be set.") - - def clean_ui_subdir(directory: Path): if directory.is_dir(): logger.info("Removing UI directory content.") diff --git a/freqtrade/templates/base_hyperopt.py.j2 b/freqtrade/templates/base_hyperopt.py.j2 deleted file mode 100644 index f6ca1477a..000000000 --- a/freqtrade/templates/base_hyperopt.py.j2 +++ /dev/null @@ -1,137 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -# --- Do not remove these libs --- -from functools import reduce -from typing import Any, Callable, Dict, List - -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer, Real # noqa - -from freqtrade.optimize.hyperopt_interface import IHyperOpt - -# -------------------------------- -# Add your lib to import here -import talib.abstract as ta # noqa -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class {{ hyperopt }}(IHyperOpt): - """ - This is a Hyperopt template to get you started. - - More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ - - You should: - - Add any lib you need to build your hyperopt. - - You must keep: - - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - - The methods roi_space, generate_roi_table and stoploss_space are not required - and are provided by default. - However, you may override them if you need 'roi' and 'stoploss' spaces that - differ from the defaults offered by Freqtrade. - Sample implementation of these methods will be copied to `user_data/hyperopts` when - creating the user-data directory using `freqtrade create-userdir --userdir user_data`, - or is available online under the following URL: - https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. - """ - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - {{ buy_space | indent(12) }} - ] - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - {{ buy_guards | indent(12) }} - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - # Check that the candle had volume - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - {{ sell_space | indent(12) }} - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by Hyperopt. - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - {{ sell_guards | indent(12) }} - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - # Check that the candle had volume - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend - diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 deleted file mode 100644 index 5b967f4ed..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_guards_full.j2 +++ /dev/null @@ -1,8 +0,0 @@ -if params.get('mfi-enabled'): - conditions.append(dataframe['mfi'] < params['mfi-value']) -if params.get('fastd-enabled'): - conditions.append(dataframe['fastd'] < params['fastd-value']) -if params.get('adx-enabled'): - conditions.append(dataframe['adx'] > params['adx-value']) -if params.get('rsi-enabled'): - conditions.append(dataframe['rsi'] < params['rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 deleted file mode 100644 index 5e1022f59..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_guards_minimal.j2 +++ /dev/null @@ -1,2 +0,0 @@ -if params.get('rsi-enabled'): - conditions.append(dataframe['rsi'] < params['rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 deleted file mode 100644 index 29bafbd93..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_space_full.j2 +++ /dev/null @@ -1,9 +0,0 @@ -Integer(10, 25, name='mfi-value'), -Integer(15, 45, name='fastd-value'), -Integer(20, 50, name='adx-value'), -Integer(20, 40, name='rsi-value'), -Categorical([True, False], name='mfi-enabled'), -Categorical([True, False], name='fastd-enabled'), -Categorical([True, False], name='adx-enabled'), -Categorical([True, False], name='rsi-enabled'), -Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') diff --git a/freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 deleted file mode 100644 index 5ddf537fb..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_buy_space_minimal.j2 +++ /dev/null @@ -1,3 +0,0 @@ -Integer(20, 40, name='rsi-value'), -Categorical([True, False], name='rsi-enabled'), -Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 deleted file mode 100644 index bd7b499f4..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_guards_full.j2 +++ /dev/null @@ -1,8 +0,0 @@ -if params.get('sell-mfi-enabled'): - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) -if params.get('sell-fastd-enabled'): - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) -if params.get('sell-adx-enabled'): - conditions.append(dataframe['adx'] < params['sell-adx-value']) -if params.get('sell-rsi-enabled'): - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 deleted file mode 100644 index 8b4adebf6..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_guards_minimal.j2 +++ /dev/null @@ -1,2 +0,0 @@ -if params.get('sell-rsi-enabled'): - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 deleted file mode 100644 index 46469d532..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_space_full.j2 +++ /dev/null @@ -1,11 +0,0 @@ -Integer(75, 100, name='sell-mfi-value'), -Integer(50, 100, name='sell-fastd-value'), -Integer(50, 100, name='sell-adx-value'), -Integer(60, 100, name='sell-rsi-value'), -Categorical([True, False], name='sell-mfi-enabled'), -Categorical([True, False], name='sell-fastd-enabled'), -Categorical([True, False], name='sell-adx-enabled'), -Categorical([True, False], name='sell-rsi-enabled'), -Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') diff --git a/freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 b/freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 deleted file mode 100644 index dfb110543..000000000 --- a/freqtrade/templates/subtemplates/hyperopt_sell_space_minimal.j2 +++ /dev/null @@ -1,5 +0,0 @@ -Integer(60, 100, name='sell-rsi-value'), -Categorical([True, False], name='sell-rsi-enabled'), -Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1da9e5100..8db2020b5 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -12,7 +12,7 @@ from freqtrade.commands import (start_convert_data, start_create_userdir, start_ start_hyperopt_list, start_hyperopt_show, start_install_ui, start_list_data, start_list_exchanges, start_list_hyperopts, start_list_markets, start_list_strategies, start_list_timeframes, - start_new_hyperopt, start_new_strategy, start_show_trades, + start_new_strategy, start_show_trades, start_test_pairlist, start_trading, start_webserver) from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) @@ -519,37 +519,6 @@ def test_start_new_strategy_no_arg(mocker, caplog): start_new_strategy(get_args(args)) -def test_start_new_hyperopt(mocker, caplog): - wt_mock = mocker.patch.object(Path, "write_text", MagicMock()) - mocker.patch.object(Path, "exists", MagicMock(return_value=False)) - - args = [ - "new-hyperopt", - "--hyperopt", - "CoolNewhyperopt" - ] - start_new_hyperopt(get_args(args)) - - assert wt_mock.call_count == 1 - assert "CoolNewhyperopt" in wt_mock.call_args_list[0][0][0] - assert log_has_re("Writing hyperopt to .*", caplog) - - mocker.patch('freqtrade.commands.deploy_commands.setup_utils_configuration') - mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - with pytest.raises(OperationalException, - match=r".* already exists. Please choose another Hyperopt Name\."): - start_new_hyperopt(get_args(args)) - - -def test_start_new_hyperopt_no_arg(mocker): - args = [ - "new-hyperopt", - ] - with pytest.raises(OperationalException, - match="`new-hyperopt` requires --hyperopt to be set."): - start_new_hyperopt(get_args(args)) - - def test_start_install_ui(mocker): clean_mock = mocker.patch('freqtrade.commands.deploy_commands.clean_ui_subdir') get_url_mock = mocker.patch('freqtrade.commands.deploy_commands.get_ui_download_url', From dad4a49e81c3a01eb98166b6db20fb08bb1a3a9a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 09:06:57 +0200 Subject: [PATCH 49/58] Remove legacy hyperopt interface from hyperopt.py --- freqtrade/commands/cli_options.py | 2 +- freqtrade/optimize/hyperopt.py | 64 ++--- freqtrade/optimize/hyperopt_auto.py | 21 +- freqtrade/optimize/hyperopt_interface.py | 14 +- freqtrade/resolvers/hyperopt_resolver.py | 38 --- freqtrade/templates/sample_hyperopt.py | 174 ----------- .../templates/sample_hyperopt_advanced.py | 269 ------------------ 7 files changed, 25 insertions(+), 557 deletions(-) delete mode 100644 freqtrade/templates/sample_hyperopt.py delete mode 100644 freqtrade/templates/sample_hyperopt_advanced.py diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index cf7cb804c..a1790cb9a 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -209,7 +209,7 @@ AVAILABLE_CLI_OPTIONS = { ), "hyperopt_path": Arg( '--hyperopt-path', - help='Specify additional lookup path for Hyperopt and Hyperopt Loss functions.', + help='Specify additional lookup path for Hyperopt Loss functions.', metavar='PATH', ), "epochs": Arg( diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index e0b35df32..d37c68769 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -22,6 +22,7 @@ from pandas import DataFrame from freqtrade.constants import DATETIME_PRINT_FORMAT, FTHYPT_FILEVERSION, LAST_BT_RESULT_FN from freqtrade.data.converter import trim_dataframes from freqtrade.data.history import get_timerange +from freqtrade.exceptions import OperationalException from freqtrade.misc import deep_merge_dicts, file_dump_json, plural from freqtrade.optimize.backtesting import Backtesting # Import IHyperOpt and IHyperOptLoss to allow unpickling classes from these modules @@ -30,7 +31,7 @@ from freqtrade.optimize.hyperopt_interface import IHyperOpt # noqa: F401 from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss # noqa: F401 from freqtrade.optimize.hyperopt_tools import HyperoptTools, hyperopt_serializer from freqtrade.optimize.optimize_reports import generate_strategy_stats -from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver, HyperOptResolver +from freqtrade.resolvers.hyperopt_resolver import HyperOptLossResolver # Suppress scikit-learn FutureWarnings from skopt @@ -80,8 +81,9 @@ class Hyperopt: self.custom_hyperopt = HyperOptAuto(self.config) self.auto_hyperopt = True else: - self.custom_hyperopt = HyperOptResolver.load_hyperopt(self.config) - self.auto_hyperopt = False + raise OperationalException( + "Using seperate Hyperopt files has been removed in 2021.9. Please convert " + "your existing Hyperopt file to the new Hyperoptable strategy interface") self.backtesting._set_strategy(self.backtesting.strategylist[0]) self.custom_hyperopt.strategy = self.backtesting.strategy @@ -103,31 +105,6 @@ class Hyperopt: self.num_epochs_saved = 0 self.current_best_epoch: Optional[Dict[str, Any]] = None - if not self.auto_hyperopt: - # Populate "fallback" functions here - # (hasattr is slow so should not be run during "regular" operations) - if hasattr(self.custom_hyperopt, 'populate_indicators'): - logger.warning( - "DEPRECATED: Using `populate_indicators()` in the hyperopt file is deprecated. " - "Please move these methods to your strategy." - ) - self.backtesting.strategy.populate_indicators = ( # type: ignore - self.custom_hyperopt.populate_indicators) # type: ignore - if hasattr(self.custom_hyperopt, 'populate_buy_trend'): - logger.warning( - "DEPRECATED: Using `populate_buy_trend()` in the hyperopt file is deprecated. " - "Please move these methods to your strategy." - ) - self.backtesting.strategy.populate_buy_trend = ( # type: ignore - self.custom_hyperopt.populate_buy_trend) # type: ignore - if hasattr(self.custom_hyperopt, 'populate_sell_trend'): - logger.warning( - "DEPRECATED: Using `populate_sell_trend()` in the hyperopt file is deprecated. " - "Please move these methods to your strategy." - ) - self.backtesting.strategy.populate_sell_trend = ( # type: ignore - self.custom_hyperopt.populate_sell_trend) # type: ignore - # Use max_open_trades for hyperopt as well, except --disable-max-market-positions is set if self.config.get('use_max_market_positions', True): self.max_open_trades = self.config['max_open_trades'] @@ -256,7 +233,7 @@ class Hyperopt: """ Assign the dimensions in the hyperoptimization space. """ - if self.auto_hyperopt and HyperoptTools.has_space(self.config, 'protection'): + if HyperoptTools.has_space(self.config, 'protection'): # Protections can only be optimized when using the Parameter interface logger.debug("Hyperopt has 'protection' space") # Enable Protections if protection space is selected. @@ -285,6 +262,15 @@ class Hyperopt: self.dimensions = (self.buy_space + self.sell_space + self.protection_space + self.roi_space + self.stoploss_space + self.trailing_space) + def assign_params(self, params_dict: Dict, category: str) -> None: + """ + Assign hyperoptable parameters + """ + for attr_name, attr in self.backtesting.strategy.enumerate_parameters(category): + if attr.optimize: + # noinspection PyProtectedMember + attr.value = params_dict[attr_name] + def generate_optimizer(self, raw_params: List[Any], iteration=None) -> Dict: """ Used Optimize function. @@ -296,18 +282,13 @@ class Hyperopt: # Apply parameters if HyperoptTools.has_space(self.config, 'buy'): - self.backtesting.strategy.advise_buy = ( # type: ignore - self.custom_hyperopt.buy_strategy_generator(params_dict)) + self.assign_params(params_dict, 'buy') if HyperoptTools.has_space(self.config, 'sell'): - self.backtesting.strategy.advise_sell = ( # type: ignore - self.custom_hyperopt.sell_strategy_generator(params_dict)) + self.assign_params(params_dict, 'sell') if HyperoptTools.has_space(self.config, 'protection'): - for attr_name, attr in self.backtesting.strategy.enumerate_parameters('protection'): - if attr.optimize: - # noinspection PyProtectedMember - attr.value = params_dict[attr_name] + self.assign_params(params_dict, 'protection') if HyperoptTools.has_space(self.config, 'roi'): self.backtesting.strategy.minimal_roi = ( # type: ignore @@ -517,11 +498,10 @@ class Hyperopt: f"saved to '{self.results_file}'.") if self.current_best_epoch: - if self.auto_hyperopt: - HyperoptTools.try_export_params( - self.config, - self.backtesting.strategy.get_strategy_name(), - self.current_best_epoch) + HyperoptTools.try_export_params( + self.config, + self.backtesting.strategy.get_strategy_name(), + self.current_best_epoch) HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, self.print_json) diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 43e92d9c6..022f04a84 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -22,26 +22,6 @@ class HyperOptAuto(IHyperOpt): sell_indicator_space methods, but other hyperopt methods can be overridden as well. """ - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: - def populate_buy_trend(dataframe: DataFrame, metadata: dict): - for attr_name, attr in self.strategy.enumerate_parameters('buy'): - if attr.optimize: - # noinspection PyProtectedMember - attr.value = params[attr_name] - return self.strategy.populate_buy_trend(dataframe, metadata) - - return populate_buy_trend - - def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable: - def populate_sell_trend(dataframe: DataFrame, metadata: dict): - for attr_name, attr in self.strategy.enumerate_parameters('sell'): - if attr.optimize: - # noinspection PyProtectedMember - attr.value = params[attr_name] - return self.strategy.populate_sell_trend(dataframe, metadata) - - return populate_sell_trend - def _get_func(self, name) -> Callable: """ Return a function defined in Strategy.HyperOpt class, or one defined in super() class. @@ -61,6 +41,7 @@ class HyperOptAuto(IHyperOpt): yield attr.get_space(attr_name) def _get_indicator_space(self, category, fallback_method_name): + # TODO: is this necessary, or can we call "generate_space" directly? indicator_space = list(self._generate_indicator_space(category)) if len(indicator_space) > 0: return indicator_space diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 500798627..814260f5e 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -5,7 +5,7 @@ This module defines the interface to apply for hyperopt import logging import math from abc import ABC -from typing import Any, Callable, Dict, List +from typing import Dict, List from skopt.space import Categorical, Dimension, Integer @@ -45,18 +45,6 @@ class IHyperOpt(ABC): IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED IHyperOpt.timeframe = str(config['timeframe']) - def buy_strategy_generator(self, params: Dict[str, Any]) -> Callable: - """ - Create a buy strategy generator. - """ - raise OperationalException(_format_exception_message('buy_strategy_generator', 'buy')) - - def sell_strategy_generator(self, params: Dict[str, Any]) -> Callable: - """ - Create a sell strategy generator. - """ - raise OperationalException(_format_exception_message('sell_strategy_generator', 'sell')) - def protection_space(self) -> List[Dimension]: """ Create a protection space. diff --git a/freqtrade/resolvers/hyperopt_resolver.py b/freqtrade/resolvers/hyperopt_resolver.py index 8327a4d13..6f0263e93 100644 --- a/freqtrade/resolvers/hyperopt_resolver.py +++ b/freqtrade/resolvers/hyperopt_resolver.py @@ -9,7 +9,6 @@ from typing import Dict from freqtrade.constants import HYPEROPT_LOSS_BUILTIN, USERPATH_HYPEROPTS from freqtrade.exceptions import OperationalException -from freqtrade.optimize.hyperopt_interface import IHyperOpt from freqtrade.optimize.hyperopt_loss_interface import IHyperOptLoss from freqtrade.resolvers import IResolver @@ -17,43 +16,6 @@ from freqtrade.resolvers import IResolver logger = logging.getLogger(__name__) -class HyperOptResolver(IResolver): - """ - This class contains all the logic to load custom hyperopt class - """ - object_type = IHyperOpt - object_type_str = "Hyperopt" - user_subdir = USERPATH_HYPEROPTS - initial_search_path = None - - @staticmethod - def load_hyperopt(config: Dict) -> IHyperOpt: - """ - Load the custom hyperopt class from config parameter - :param config: configuration dictionary - """ - if not config.get('hyperopt'): - raise OperationalException("No Hyperopt set. Please use `--hyperopt` to specify " - "the Hyperopt class to use.") - - hyperopt_name = config['hyperopt'] - - hyperopt = HyperOptResolver.load_object(hyperopt_name, config, - kwargs={'config': config}, - extra_dir=config.get('hyperopt_path')) - - if not hasattr(hyperopt, 'populate_indicators'): - logger.info("Hyperopt class does not provide populate_indicators() method. " - "Using populate_indicators from the strategy.") - if not hasattr(hyperopt, 'populate_buy_trend'): - logger.info("Hyperopt class does not provide populate_buy_trend() method. " - "Using populate_buy_trend from the strategy.") - if not hasattr(hyperopt, 'populate_sell_trend'): - logger.info("Hyperopt class does not provide populate_sell_trend() method. " - "Using populate_sell_trend from the strategy.") - return hyperopt - - class HyperOptLossResolver(IResolver): """ This class contains all the logic to load custom hyperopt loss class diff --git a/freqtrade/templates/sample_hyperopt.py b/freqtrade/templates/sample_hyperopt.py deleted file mode 100644 index ed1af7718..000000000 --- a/freqtrade/templates/sample_hyperopt.py +++ /dev/null @@ -1,174 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement -# isort: skip_file - -# --- Do not remove these libs --- -from functools import reduce -from typing import Any, Callable, Dict, List - -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer, Real # noqa - -from freqtrade.optimize.hyperopt_interface import IHyperOpt - -# -------------------------------- -# Add your lib to import here -import talib.abstract as ta # noqa -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class SampleHyperOpt(IHyperOpt): - """ - This is a sample Hyperopt to inspire you. - - More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ - - You should: - - Rename the class name to some unique name. - - Add any methods you want to build your hyperopt. - - Add any lib you need to build your hyperopt. - - An easier way to get a new hyperopt file is by using - `freqtrade new-hyperopt --hyperopt MyCoolHyperopt`. - - You must keep: - - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - - The methods roi_space, generate_roi_table and stoploss_space are not required - and are provided by default. - However, you may override them if you need 'roi' and 'stoploss' spaces that - differ from the defaults offered by Freqtrade. - Sample implementation of these methods will be copied to `user_data/hyperopts` when - creating the user-data directory using `freqtrade create-userdir --userdir user_data`, - or is available online under the following URL: - https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py. - """ - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by Hyperopt. - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) - if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) - if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] < params['sell-adx-value']) - if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend diff --git a/freqtrade/templates/sample_hyperopt_advanced.py b/freqtrade/templates/sample_hyperopt_advanced.py deleted file mode 100644 index cc13b6ba3..000000000 --- a/freqtrade/templates/sample_hyperopt_advanced.py +++ /dev/null @@ -1,269 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement -# isort: skip_file -# --- Do not remove these libs --- -from functools import reduce -from typing import Any, Callable, Dict, List - -import numpy as np # noqa -import pandas as pd # noqa -from pandas import DataFrame -from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Real # noqa - -from freqtrade.optimize.hyperopt_interface import IHyperOpt - -# -------------------------------- -# Add your lib to import here -import talib.abstract as ta # noqa -import freqtrade.vendor.qtpylib.indicators as qtpylib - - -class AdvancedSampleHyperOpt(IHyperOpt): - """ - This is a sample hyperopt to inspire you. - Feel free to customize it. - - More information in the documentation: https://www.freqtrade.io/en/latest/hyperopt/ - - You should: - - Rename the class name to some unique name. - - Add any methods you want to build your hyperopt. - - Add any lib you need to build your hyperopt. - - You must keep: - - The prototypes for the methods: populate_indicators, indicator_space, buy_strategy_generator. - - The methods roi_space, generate_roi_table and stoploss_space are not required - and are provided by default. - However, you may override them if you need the - 'roi' and the 'stoploss' spaces that differ from the defaults offered by Freqtrade. - - This sample illustrates how to override these methods. - """ - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - This method can also be loaded from the strategy, if it doesn't exist in the hyperopt class. - """ - dataframe['adx'] = ta.ADX(dataframe) - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - dataframe['mfi'] = ta.MFI(dataframe) - dataframe['rsi'] = ta.RSI(dataframe) - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_upperband'] = bollinger['upper'] - dataframe['sar'] = ta.SAR(dataframe) - return dataframe - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by hyperopt - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use - """ - conditions = [] - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by hyperopt - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use - """ - # print(params) - conditions = [] - # GUARDS AND TRENDS - if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) - if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) - if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] < params['sell-adx-value']) - if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend - - @staticmethod - def generate_roi_table(params: Dict) -> Dict[int, float]: - """ - Generate the ROI table that will be used by Hyperopt - - This implementation generates the default legacy Freqtrade ROI tables. - - Change it if you need different number of steps in the generated - ROI tables or other structure of the ROI tables. - - Please keep it aligned with parameters in the 'roi' optimization - hyperspace defined by the roi_space method. - """ - roi_table = {} - roi_table[0] = params['roi_p1'] + params['roi_p2'] + params['roi_p3'] - roi_table[params['roi_t3']] = params['roi_p1'] + params['roi_p2'] - roi_table[params['roi_t3'] + params['roi_t2']] = params['roi_p1'] - roi_table[params['roi_t3'] + params['roi_t2'] + params['roi_t1']] = 0 - - return roi_table - - @staticmethod - def roi_space() -> List[Dimension]: - """ - Values to search for each ROI steps - - Override it if you need some different ranges for the parameters in the - 'roi' optimization hyperspace. - - Please keep it aligned with the implementation of the - generate_roi_table method. - """ - return [ - Integer(10, 120, name='roi_t1'), - Integer(10, 60, name='roi_t2'), - Integer(10, 40, name='roi_t3'), - SKDecimal(0.01, 0.04, decimals=3, name='roi_p1'), - SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'), - SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'), - ] - - @staticmethod - def stoploss_space() -> List[Dimension]: - """ - Stoploss Value to search - - Override it if you need some different range for the parameter in the - 'stoploss' optimization hyperspace. - """ - return [ - SKDecimal(-0.35, -0.02, decimals=3, name='stoploss'), - ] - - @staticmethod - def trailing_space() -> List[Dimension]: - """ - Create a trailing stoploss space. - - You may override it in your custom Hyperopt class. - """ - return [ - # It was decided to always set trailing_stop is to True if the 'trailing' hyperspace - # is used. Otherwise hyperopt will vary other parameters that won't have effect if - # trailing_stop is set False. - # This parameter is included into the hyperspace dimensions rather than assigning - # it explicitly in the code in order to have it printed in the results along with - # other 'trailing' hyperspace parameters. - Categorical([True], name='trailing_stop'), - - SKDecimal(0.01, 0.35, decimals=3, name='trailing_stop_positive'), - - # 'trailing_stop_positive_offset' should be greater than 'trailing_stop_positive', - # so this intermediate parameter is used as the value of the difference between - # them. The value of the 'trailing_stop_positive_offset' is constructed in the - # generate_trailing_params() method. - # This is similar to the hyperspace dimensions used for constructing the ROI tables. - SKDecimal(0.001, 0.1, decimals=3, name='trailing_stop_positive_offset_p1'), - - Categorical([True, False], name='trailing_only_offset_is_reached'), - ] From fd6bf591f8abb888e3ad340bcc9c368096ae92c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 17:11:02 +0200 Subject: [PATCH 50/58] Update some tests to remove explicit hyperopt interface --- freqtrade/optimize/hyperopt.py | 3 +- freqtrade/optimize/hyperopt_auto.py | 22 ++- freqtrade/optimize/hyperopt_interface.py | 27 --- tests/commands/test_commands.py | 43 ++--- tests/optimize/conftest.py | 2 +- tests/optimize/test_hyperopt.py | 203 ++++------------------- 6 files changed, 63 insertions(+), 237 deletions(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d37c68769..14b155546 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -79,10 +79,9 @@ class Hyperopt: if not self.config.get('hyperopt'): self.custom_hyperopt = HyperOptAuto(self.config) - self.auto_hyperopt = True else: raise OperationalException( - "Using seperate Hyperopt files has been removed in 2021.9. Please convert " + "Using separate Hyperopt files has been removed in 2021.9. Please convert " "your existing Hyperopt file to the new Hyperoptable strategy interface") self.backtesting._set_strategy(self.backtesting.strategylist[0]) diff --git a/freqtrade/optimize/hyperopt_auto.py b/freqtrade/optimize/hyperopt_auto.py index 022f04a84..1f11cec80 100644 --- a/freqtrade/optimize/hyperopt_auto.py +++ b/freqtrade/optimize/hyperopt_auto.py @@ -4,9 +4,9 @@ This module implements a convenience auto-hyperopt class, which can be used toge that implement IHyperStrategy interface. """ from contextlib import suppress -from typing import Any, Callable, Dict, List +from typing import Callable, Dict, List -from pandas import DataFrame +from freqtrade.exceptions import OperationalException with suppress(ImportError): @@ -15,6 +15,14 @@ with suppress(ImportError): from freqtrade.optimize.hyperopt_interface import IHyperOpt +def _format_exception_message(space: str) -> str: + raise OperationalException( + f"The '{space}' space is included into the hyperoptimization " + f"but no parameter for this space was not found in your Strategy. " + f"Please make sure to have parameters for this space enabled for optimization " + f"or remove the '{space}' space from hyperoptimization.") + + class HyperOptAuto(IHyperOpt): """ This class delegates functionality to Strategy(IHyperStrategy) and Strategy.HyperOpt classes. @@ -40,22 +48,22 @@ class HyperOptAuto(IHyperOpt): if attr.optimize: yield attr.get_space(attr_name) - def _get_indicator_space(self, category, fallback_method_name): + def _get_indicator_space(self, category): # TODO: is this necessary, or can we call "generate_space" directly? indicator_space = list(self._generate_indicator_space(category)) if len(indicator_space) > 0: return indicator_space else: - return self._get_func(fallback_method_name)() + _format_exception_message(category) def indicator_space(self) -> List['Dimension']: - return self._get_indicator_space('buy', 'indicator_space') + return self._get_indicator_space('buy') def sell_indicator_space(self) -> List['Dimension']: - return self._get_indicator_space('sell', 'sell_indicator_space') + return self._get_indicator_space('sell') def protection_space(self) -> List['Dimension']: - return self._get_indicator_space('protection', 'protection_space') + return self._get_indicator_space('protection') def generate_roi_table(self, params: Dict) -> Dict[int, float]: return self._get_func('generate_roi_table')(params) diff --git a/freqtrade/optimize/hyperopt_interface.py b/freqtrade/optimize/hyperopt_interface.py index 814260f5e..8fb40f557 100644 --- a/freqtrade/optimize/hyperopt_interface.py +++ b/freqtrade/optimize/hyperopt_interface.py @@ -9,7 +9,6 @@ from typing import Dict, List from skopt.space import Categorical, Dimension, Integer -from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import round_dict from freqtrade.optimize.space import SKDecimal @@ -19,13 +18,6 @@ from freqtrade.strategy import IStrategy logger = logging.getLogger(__name__) -def _format_exception_message(method: str, space: str) -> str: - return (f"The '{space}' space is included into the hyperoptimization " - f"but {method}() method is not found in your " - f"custom Hyperopt class. You should either implement this " - f"method or remove the '{space}' space from hyperoptimization.") - - class IHyperOpt(ABC): """ Interface for freqtrade hyperopt @@ -45,25 +37,6 @@ class IHyperOpt(ABC): IHyperOpt.ticker_interval = str(config['timeframe']) # DEPRECATED IHyperOpt.timeframe = str(config['timeframe']) - def protection_space(self) -> List[Dimension]: - """ - Create a protection space. - Only supported by the Parameter interface. - """ - raise OperationalException(_format_exception_message('indicator_space', 'protection')) - - def indicator_space(self) -> List[Dimension]: - """ - Create an indicator space. - """ - raise OperationalException(_format_exception_message('indicator_space', 'buy')) - - def sell_indicator_space(self) -> List[Dimension]: - """ - Create a sell indicator space. - """ - raise OperationalException(_format_exception_message('sell_indicator_space', 'sell')) - def generate_roi_table(self, params: Dict) -> Dict[int, float]: """ Create a ROI table. diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 8db2020b5..a6a87d245 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -10,10 +10,10 @@ import pytest from freqtrade.commands import (start_convert_data, start_create_userdir, start_download_data, start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, start_list_timeframes, - start_new_strategy, start_show_trades, - start_test_pairlist, start_trading, start_webserver) + start_list_data, start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, start_new_strategy, + start_show_trades, start_test_pairlist, start_trading, + start_webserver) from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration @@ -793,37 +793,20 @@ def test_start_list_strategies(mocker, caplog, capsys): assert "legacy_strategy_v1.py" in captured.out assert "StrategyTestV2" in captured.out - -def test_start_list_hyperopts(mocker, caplog, capsys): - + # Test color output args = [ - "list-hyperopts", - "--hyperopt-path", - str(Path(__file__).parent.parent / "optimize" / "hyperopts"), - "-1" + "list-strategies", + "--strategy-path", + str(Path(__file__).parent.parent / "strategy" / "strats"), ] pargs = get_args(args) # pargs['config'] = None - start_list_hyperopts(pargs) + start_list_strategies(pargs) captured = capsys.readouterr() - assert "TestHyperoptLegacy" not in captured.out - assert "legacy_hyperopt.py" not in captured.out - assert "HyperoptTestSepFile" in captured.out - assert "test_hyperopt.py" not in captured.out - - # Test regular output - args = [ - "list-hyperopts", - "--hyperopt-path", - str(Path(__file__).parent.parent / "optimize" / "hyperopts"), - ] - pargs = get_args(args) - # pargs['config'] = None - start_list_hyperopts(pargs) - captured = capsys.readouterr() - assert "TestHyperoptLegacy" not in captured.out - assert "legacy_hyperopt.py" not in captured.out - assert "HyperoptTestSepFile" in captured.out + assert "TestStrategyLegacyV1" in captured.out + assert "legacy_strategy_v1.py" in captured.out + assert "StrategyTestV2" in captured.out + assert "LOAD FAILED" in captured.out def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 95c9fef97..5c5171c3a 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -16,7 +16,7 @@ def hyperopt_conf(default_conf): hyperconf.update({ 'datadir': Path(default_conf['datadir']), 'runmode': RunMode.HYPEROPT, - 'hyperopt': 'HyperoptTestSepFile', + 'strategy': 'HyperoptableStrategy', 'hyperopt_loss': 'ShortTradeDurHyperOptLoss', 'hyperopt_path': str(Path(__file__).parent / 'hyperopts'), 'epochs': 1, diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 565d6077a..498b6e588 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -17,7 +17,6 @@ from freqtrade.optimize.hyperopt_auto import HyperOptAuto from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.optimize.space import SKDecimal -from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.strategy.hyper import IntParameter from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) @@ -133,47 +132,6 @@ def test_setup_hyperopt_configuration_stake_amount(mocker, default_conf) -> None setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) -def test_hyperoptresolver(mocker, default_conf, caplog) -> None: - patched_configuration_load_config_file(mocker, default_conf) - - hyperopt = HyperoptTestSepFile - delattr(hyperopt, 'populate_indicators') - delattr(hyperopt, 'populate_buy_trend') - delattr(hyperopt, 'populate_sell_trend') - mocker.patch( - 'freqtrade.resolvers.hyperopt_resolver.HyperOptResolver.load_object', - MagicMock(return_value=hyperopt(default_conf)) - ) - default_conf.update({'hyperopt': 'HyperoptTestSepFile'}) - x = HyperOptResolver.load_hyperopt(default_conf) - assert not hasattr(x, 'populate_indicators') - assert not hasattr(x, 'populate_buy_trend') - assert not hasattr(x, 'populate_sell_trend') - assert log_has("Hyperopt class does not provide populate_indicators() method. " - "Using populate_indicators from the strategy.", caplog) - assert log_has("Hyperopt class does not provide populate_sell_trend() method. " - "Using populate_sell_trend from the strategy.", caplog) - assert log_has("Hyperopt class does not provide populate_buy_trend() method. " - "Using populate_buy_trend from the strategy.", caplog) - assert hasattr(x, "ticker_interval") # DEPRECATED - assert hasattr(x, "timeframe") - - -def test_hyperoptresolver_wrongname(default_conf) -> None: - default_conf.update({'hyperopt': "NonExistingHyperoptClass"}) - - with pytest.raises(OperationalException, match=r'Impossible to load Hyperopt.*'): - HyperOptResolver.load_hyperopt(default_conf) - - -def test_hyperoptresolver_noname(default_conf): - default_conf['hyperopt'] = '' - with pytest.raises(OperationalException, - match="No Hyperopt set. Please use `--hyperopt` to specify " - "the Hyperopt class to use."): - HyperOptResolver.load_hyperopt(default_conf) - - def test_start_not_installed(mocker, default_conf, import_fails) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, default_conf) @@ -196,7 +154,7 @@ def test_start_not_installed(mocker, default_conf, import_fails) -> None: start_hyperopt(pargs) -def test_start(mocker, hyperopt_conf, caplog) -> None: +def test_start_no_hyperopt_allowed(mocker, hyperopt_conf, caplog) -> None: start_mock = MagicMock() patched_configuration_load_config_file(mocker, hyperopt_conf) mocker.patch('freqtrade.optimize.hyperopt.Hyperopt.start', start_mock) @@ -210,10 +168,8 @@ def test_start(mocker, hyperopt_conf, caplog) -> None: '--epochs', '5' ] pargs = get_args(args) - start_hyperopt(pargs) - - assert log_has('Starting freqtrade in Hyperopt mode', caplog) - assert start_mock.call_count == 1 + with pytest.raises(OperationalException, match=r"Using separate Hyperopt files has been.*"): + start_hyperopt(pargs) def test_start_no_data(mocker, hyperopt_conf) -> None: @@ -225,11 +181,11 @@ def test_start_no_data(mocker, hyperopt_conf) -> None: ) patch_exchange(mocker) - + # TODO: migrate to strategy-based hyperopt args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--hyperopt-loss', 'SharpeHyperOptLossDaily', '--epochs', '5' ] @@ -427,66 +383,14 @@ def test_hyperopt_format_results(hyperopt): def test_populate_indicators(hyperopt, testdatadir) -> None: data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) - dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], - {'pair': 'UNITTEST/BTC'}) + dataframe = dataframes['UNITTEST/BTC'] # Check if some indicators are generated. We will not test all of them assert 'adx' in dataframe - assert 'mfi' in dataframe + assert 'macd' in dataframe assert 'rsi' in dataframe -def test_buy_strategy_generator(hyperopt, testdatadir) -> None: - data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) - dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) - dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], - {'pair': 'UNITTEST/BTC'}) - - populate_buy_trend = hyperopt.custom_hyperopt.buy_strategy_generator( - { - 'adx-value': 20, - 'fastd-value': 20, - 'mfi-value': 20, - 'rsi-value': 20, - 'adx-enabled': True, - 'fastd-enabled': True, - 'mfi-enabled': True, - 'rsi-enabled': True, - 'trigger': 'bb_lower' - } - ) - result = populate_buy_trend(dataframe, {'pair': 'UNITTEST/BTC'}) - # Check if some indicators are generated. We will not test all of them - assert 'buy' in result - assert 1 in result['buy'] - - -def test_sell_strategy_generator(hyperopt, testdatadir) -> None: - data = load_data(testdatadir, '1m', ['UNITTEST/BTC'], fill_up_missing=True) - dataframes = hyperopt.backtesting.strategy.advise_all_indicators(data) - dataframe = hyperopt.custom_hyperopt.populate_indicators(dataframes['UNITTEST/BTC'], - {'pair': 'UNITTEST/BTC'}) - - populate_sell_trend = hyperopt.custom_hyperopt.sell_strategy_generator( - { - 'sell-adx-value': 20, - 'sell-fastd-value': 75, - 'sell-mfi-value': 80, - 'sell-rsi-value': 20, - 'sell-adx-enabled': True, - 'sell-fastd-enabled': True, - 'sell-mfi-enabled': True, - 'sell-rsi-enabled': True, - 'sell-trigger': 'sell-bb_upper' - } - ) - result = populate_sell_trend(dataframe, {'pair': 'UNITTEST/BTC'}) - # Check if some indicators are generated. We will not test all of them - print(result) - assert 'sell' in result - assert 1 in result['sell'] - - def test_generate_optimizer(mocker, hyperopt_conf) -> None: hyperopt_conf.update({'spaces': 'all', 'hyperopt_min_trades': 1, @@ -527,24 +431,12 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: mocker.patch('freqtrade.optimize.hyperopt.load', return_value={'XRP/BTC': None}) optimizer_param = { - 'adx-value': 0, - 'fastd-value': 35, - 'mfi-value': 0, - 'rsi-value': 0, - 'adx-enabled': False, - 'fastd-enabled': True, - 'mfi-enabled': False, - 'rsi-enabled': False, - 'trigger': 'macd_cross_signal', - 'sell-adx-value': 0, - 'sell-fastd-value': 75, - 'sell-mfi-value': 0, - 'sell-rsi-value': 0, - 'sell-adx-enabled': False, - 'sell-fastd-enabled': True, - 'sell-mfi-enabled': False, - 'sell-rsi-enabled': False, - 'sell-trigger': 'macd_cross_signal', + 'buy_plusdi': 0.02, + 'buy_rsi': 35, + 'sell_minusdi': 0.02, + 'sell_rsi': 75, + 'protection_cooldown_lookback': 20, + 'protection_enabled': True, 'roi_t1': 60.0, 'roi_t2': 30.0, 'roi_t3': 20.0, @@ -564,29 +456,19 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: '0.00003100 BTC ( 0.00%). ' 'Avg duration 0:50:00 min.' ), - 'params_details': {'buy': {'adx-enabled': False, - 'adx-value': 0, - 'fastd-enabled': True, - 'fastd-value': 35, - 'mfi-enabled': False, - 'mfi-value': 0, - 'rsi-enabled': False, - 'rsi-value': 0, - 'trigger': 'macd_cross_signal'}, + 'params_details': {'buy': {'buy_plusdi': 0.02, + 'buy_rsi': 35, + }, 'roi': {"0": 0.12000000000000001, "20.0": 0.02, "50.0": 0.01, "110.0": 0}, - 'protection': {}, - 'sell': {'sell-adx-enabled': False, - 'sell-adx-value': 0, - 'sell-fastd-enabled': True, - 'sell-fastd-value': 75, - 'sell-mfi-enabled': False, - 'sell-mfi-value': 0, - 'sell-rsi-enabled': False, - 'sell-rsi-value': 0, - 'sell-trigger': 'macd_cross_signal'}, + 'protection': {'protection_cooldown_lookback': 20, + 'protection_enabled': True, + }, + 'sell': {'sell_minusdi': 0.02, + 'sell_rsi': 75, + }, 'stoploss': {'stoploss': -0.4}, 'trailing': {'trailing_only_offset_is_reached': False, 'trailing_stop': True, @@ -808,11 +690,6 @@ def test_simplified_interface_roi_stoploss(mocker, hyperopt_conf, capsys) -> Non hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator - del hyperopt.custom_hyperopt.__class__.indicator_space - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - hyperopt.start() parallel.assert_called_once() @@ -843,16 +720,14 @@ def test_simplified_interface_all_failed(mocker, hyperopt_conf) -> None: hyperopt_conf.update({'spaces': 'all', }) + mocker.patch('freqtrade.optimize.hyperopt_auto.HyperOptAuto._generate_indicator_space', + return_value=[]) + hyperopt = Hyperopt(hyperopt_conf) hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator - del hyperopt.custom_hyperopt.__class__.indicator_space - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - - with pytest.raises(OperationalException, match=r"The 'buy' space is included into *"): + with pytest.raises(OperationalException, match=r"The 'protection' space is included into *"): hyperopt.start() @@ -889,11 +764,6 @@ def test_simplified_interface_buy(mocker, hyperopt_conf, capsys) -> None: hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - # TODO: sell_strategy_generator() is actually not called because - # run_optimizer_parallel() is mocked - del hyperopt.custom_hyperopt.__class__.sell_strategy_generator - del hyperopt.custom_hyperopt.__class__.sell_indicator_space - hyperopt.start() parallel.assert_called_once() @@ -943,11 +813,6 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - # TODO: buy_strategy_generator() is actually not called because - # run_optimizer_parallel() is mocked - del hyperopt.custom_hyperopt.__class__.buy_strategy_generator - del hyperopt.custom_hyperopt.__class__.indicator_space - hyperopt.start() parallel.assert_called_once() @@ -964,13 +829,12 @@ def test_simplified_interface_sell(mocker, hyperopt_conf, capsys) -> None: assert hasattr(hyperopt, "position_stacking") -@pytest.mark.parametrize("method,space", [ - ('buy_strategy_generator', 'buy'), - ('indicator_space', 'buy'), - ('sell_strategy_generator', 'sell'), - ('sell_indicator_space', 'sell'), +@pytest.mark.parametrize("space", [ + ('buy'), + ('sell'), + ('protection'), ]) -def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> None: +def test_simplified_interface_failed(mocker, hyperopt_conf, space) -> None: mocker.patch('freqtrade.optimize.hyperopt.dump', MagicMock()) mocker.patch('freqtrade.optimize.hyperopt.file_dump_json') mocker.patch('freqtrade.optimize.backtesting.Backtesting.load_bt_data', @@ -979,6 +843,8 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No 'freqtrade.optimize.hyperopt.get_timerange', MagicMock(return_value=(datetime(2017, 12, 10), datetime(2017, 12, 13))) ) + mocker.patch('freqtrade.optimize.hyperopt_auto.HyperOptAuto._generate_indicator_space', + return_value=[]) patch_exchange(mocker) @@ -988,8 +854,6 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No hyperopt.backtesting.strategy.advise_all_indicators = MagicMock() hyperopt.custom_hyperopt.generate_roi_table = MagicMock(return_value={}) - delattr(hyperopt.custom_hyperopt.__class__, method) - with pytest.raises(OperationalException, match=f"The '{space}' space is included into *"): hyperopt.start() @@ -999,7 +863,6 @@ def test_in_strategy_auto_hyperopt(mocker, hyperopt_conf, tmpdir, fee) -> None: mocker.patch('freqtrade.exchange.Exchange.get_fee', fee) (Path(tmpdir) / 'hyperopt_results').mkdir(parents=True) # No hyperopt needed - del hyperopt_conf['hyperopt'] hyperopt_conf.update({ 'strategy': 'HyperoptableStrategy', 'user_data_dir': Path(tmpdir), From 3675df83448ceaaed1c92ec4c8b7d3273e1ef461 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 17:52:47 +0200 Subject: [PATCH 51/58] Update documentation regarding Legacy Hyperopt --- README.md | 8 +- docs/advanced-hyperopt.md | 298 +----------------- docs/bot-usage.md | 8 +- docs/deprecated.md | 5 + docs/faq.md | 2 +- docs/hyperopt.md | 21 +- docs/utils.md | 2 - freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 4 +- freqtrade/constants.py | 2 - tests/exchange/test_exchange.py | 4 +- .../hyperopts/hyperopt_test_sep_file.py | 202 ------------ tests/optimize/test_hyperopt.py | 14 +- tests/plugins/test_pairlocks.py | 2 +- tests/test_directory_operations.py | 8 +- 15 files changed, 55 insertions(+), 527 deletions(-) delete mode 100644 tests/optimize/hyperopts/hyperopt_test_sep_file.py diff --git a/README.md b/README.md index 2c164135f..01effd7bc 100644 --- a/README.md +++ b/README.md @@ -79,22 +79,22 @@ For any other type of installation please refer to [Installation doc](https://ww ``` usage: freqtrade [-h] [-V] - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} ... Free, open source crypto trading bot positional arguments: - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} trade Trade module. create-userdir Create user-data directory. new-config Create new config - new-hyperopt Create new hyperopt new-strategy Create new strategy download-data Download backtesting data. convert-data Convert candle (OHLCV) data from one format to another. convert-trade-data Convert trade data from one format to another. + list-data List downloaded data. backtesting Backtesting module. edge Edge module. hyperopt Hyperopt module. @@ -108,8 +108,10 @@ positional arguments: list-timeframes Print available timeframes for the exchange. show-trades Show trades. test-pairlist Test your pairlist configuration. + install-ui Install FreqUI plot-dataframe Plot candles with indicators. plot-profit Generate plot showing profits. + webserver Webserver module. optional arguments: -h, --help show this help message and exit diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index 4a2bafd2e..f2f52b7dd 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -67,10 +67,10 @@ Currently, the arguments are: This function needs to return a floating point number (`float`). Smaller numbers will be interpreted as better results. The parameters and balancing for this is up to you. !!! Note - This function is called once per iteration - so please make sure to have this as optimized as possible to not slow hyperopt down unnecessarily. + This function is called once per epoch - so please make sure to have this as optimized as possible to not slow hyperopt down unnecessarily. -!!! Note - Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface later. +!!! Note "`*args` and `**kwargs`" + Please keep the arguments `*args` and `**kwargs` in the interface to allow us to extend this interface in the future. ## Overriding pre-defined spaces @@ -82,8 +82,22 @@ class MyAwesomeStrategy(IStrategy): # Define a custom stoploss space. def stoploss_space(): return [SKDecimal(-0.05, -0.01, decimals=3, name='stoploss')] + + # Define custom ROI space + def roi_space() -> List[Dimension]: + return [ + Integer(10, 120, name='roi_t1'), + Integer(10, 60, name='roi_t2'), + Integer(10, 40, name='roi_t3'), + SKDecimal(0.01, 0.04, decimals=3, name='roi_p1'), + SKDecimal(0.01, 0.07, decimals=3, name='roi_p2'), + SKDecimal(0.01, 0.20, decimals=3, name='roi_p3'), + ] ``` +!!! Note + All overrides are optional and can be mixed/matched as necessary. + ## Space options For the additional spaces, scikit-optimize (in combination with Freqtrade) provides the following space types: @@ -105,281 +119,3 @@ from freqtrade.optimize.space import Categorical, Dimension, Integer, SKDecimal, Assuming the definition of a rather small space (`SKDecimal(0.10, 0.15, decimals=2, name='xxx')`) - SKDecimal will have 5 possibilities (`[0.10, 0.11, 0.12, 0.13, 0.14, 0.15]`). A corresponding real space `Real(0.10, 0.15 name='xxx')` on the other hand has an almost unlimited number of possibilities (`[0.10, 0.010000000001, 0.010000000002, ... 0.014999999999, 0.01500000000]`). - ---- - -## Legacy Hyperopt - -This Section explains the configuration of an explicit Hyperopt file (separate to the strategy). - -!!! Warning "Deprecated / legacy mode" - Since the 2021.4 release you no longer have to write a separate hyperopt class, but all strategies can be hyperopted. - Please read the [main hyperopt page](hyperopt.md) for more details. - -### Prepare hyperopt file - -Configuring an explicit hyperopt file is similar to writing your own strategy, and many tasks will be similar. - -!!! Tip "About this page" - For this page, we will be using a fictional strategy called `AwesomeStrategy` - which will be optimized using the `AwesomeHyperopt` class. - -#### Create a Custom Hyperopt File - -The simplest way to get started is to use the following command, which will create a new hyperopt file from a template, which will be located under `user_data/hyperopts/AwesomeHyperopt.py`. - -Let assume you want a hyperopt file `AwesomeHyperopt.py`: - -``` bash -freqtrade new-hyperopt --hyperopt AwesomeHyperopt -``` - -#### Legacy Hyperopt checklist - -Checklist on all tasks / possibilities in hyperopt - -Depending on the space you want to optimize, only some of the below are required: - -* fill `buy_strategy_generator` - for buy signal optimization -* fill `indicator_space` - for buy signal optimization -* fill `sell_strategy_generator` - for sell signal optimization -* fill `sell_indicator_space` - for sell signal optimization - -!!! Note - `populate_indicators` needs to create all indicators any of thee spaces may use, otherwise hyperopt will not work. - -Optional in hyperopt - can also be loaded from a strategy (recommended): - -* `populate_indicators` - fallback to create indicators -* `populate_buy_trend` - fallback if not optimizing for buy space. should come from strategy -* `populate_sell_trend` - fallback if not optimizing for sell space. should come from strategy - -!!! Note - You always have to provide a strategy to Hyperopt, even if your custom Hyperopt class contains all methods. - Assuming the optional methods are not in your hyperopt file, please use `--strategy AweSomeStrategy` which contains these methods so hyperopt can use these methods instead. - -Rarely you may also need to override: - -* `roi_space` - for custom ROI optimization (if you need the ranges for the ROI parameters in the optimization hyperspace that differ from default) -* `generate_roi_table` - for custom ROI optimization (if you need the ranges for the values in the ROI table that differ from default or the number of entries (steps) in the ROI table which differs from the default 4 steps) -* `stoploss_space` - for custom stoploss optimization (if you need the range for the stoploss parameter in the optimization hyperspace that differs from default) -* `trailing_space` - for custom trailing stop optimization (if you need the ranges for the trailing stop parameters in the optimization hyperspace that differ from default) - -#### Defining a buy signal optimization - -Let's say you are curious: should you use MACD crossings or lower Bollinger -Bands to trigger your buys. And you also wonder should you use RSI or ADX to -help with those buy decisions. If you decide to use RSI or ADX, which values -should I use for them? So let's use hyperparameter optimization to solve this -mystery. - -We will start by defining a search space: - -```python - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching strategy parameters - """ - return [ - Integer(20, 40, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal'], name='trigger') - ] -``` - -Above definition says: I have five parameters I want you to randomly combine -to find the best combination. Two of them are integer values (`adx-value` and `rsi-value`) and I want you test in the range of values 20 to 40. -Then we have three category variables. First two are either `True` or `False`. -We use these to either enable or disable the ADX and RSI guards. -The last one we call `trigger` and use it to decide which buy trigger we want to use. - -So let's write the buy strategy generator using these values: - -```python - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - conditions = [] - # GUARDS AND TRENDS - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - - # Check that volume is not 0 - conditions.append(dataframe['volume'] > 0) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend -``` - -Hyperopt will now call `populate_buy_trend()` many times (`epochs`) with different value combinations. -It will use the given historical data and make buys based on the buy signals generated with the above function. -Based on the results, hyperopt will tell you which parameter combination produced the best results (based on the configured [loss function](#loss-functions)). - -!!! Note - The above setup expects to find ADX, RSI and Bollinger Bands in the populated indicators. - When you want to test an indicator that isn't used by the bot currently, remember to - add it to the `populate_indicators()` method in your strategy or hyperopt file. - -#### Sell optimization - -Similar to the buy-signal above, sell-signals can also be optimized. -Place the corresponding settings into the following methods - -* Inside `sell_indicator_space()` - the parameters hyperopt shall be optimizing. -* Within `sell_strategy_generator()` - populate the nested method `populate_sell_trend()` to apply the parameters. - -The configuration and rules are the same than for buy signals. -To avoid naming collisions in the search-space, please prefix all sell-spaces with `sell-`. - -### Execute Hyperopt - -Once you have updated your hyperopt configuration you can run it. -Because hyperopt tries a lot of combinations to find the best parameters it will take time to get a good result. More time usually results in better results. - -We strongly recommend to use `screen` or `tmux` to prevent any connection loss. - -```bash -freqtrade hyperopt --config config.json --hyperopt --hyperopt-loss --strategy -e 500 --spaces all -``` - -Use `` as the name of the custom hyperopt used. - -The `-e` option will set how many evaluations hyperopt will do. Since hyperopt uses Bayesian search, running too many epochs at once may not produce greater results. Experience has shown that best results are usually not improving much after 500-1000 epochs. -Doing multiple runs (executions) with a few 1000 epochs and different random state will most likely produce different results. - -The `--spaces all` option determines that all possible parameters should be optimized. Possibilities are listed below. - -!!! Note - Hyperopt will store hyperopt results with the timestamp of the hyperopt start time. - Reading commands (`hyperopt-list`, `hyperopt-show`) can use `--hyperopt-filename ` to read and display older hyperopt results. - You can find a list of filenames with `ls -l user_data/hyperopt_results/`. - -#### Running Hyperopt using methods from a strategy - -Hyperopt can reuse `populate_indicators`, `populate_buy_trend`, `populate_sell_trend` from your strategy, assuming these methods are **not** in your custom hyperopt file, and a strategy is provided. - -```bash -freqtrade hyperopt --hyperopt AwesomeHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy AwesomeStrategy -``` - -### Understand the Hyperopt Result - -Once Hyperopt is completed you can use the result to create a new strategy. -Given the following result from hyperopt: - -``` -Best result: - - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 - -Buy hyperspace params: -{ 'adx-value': 44, - 'rsi-value': 29, - 'adx-enabled': False, - 'rsi-enabled': True, - 'trigger': 'bb_lower'} -``` - -You should understand this result like: - -* The buy trigger that worked best was `bb_lower`. -* You should not use ADX because `adx-enabled: False`) -* You should **consider** using the RSI indicator (`rsi-enabled: True` and the best value is `29.0` (`rsi-value: 29.0`) - -You have to look inside your strategy file into `buy_strategy_generator()` -method, what those values match to. - -So for example you had `rsi-value: 29.0` so we would look at `rsi`-block, that translates to the following code block: - -```python -(dataframe['rsi'] < 29.0) -``` - -Translating your whole hyperopt result as the new buy-signal would then look like: - -```python -def populate_buy_trend(self, dataframe: DataFrame) -> DataFrame: - dataframe.loc[ - ( - (dataframe['rsi'] < 29.0) & # rsi-value - dataframe['close'] < dataframe['bb_lowerband'] # trigger - ), - 'buy'] = 1 - return dataframe -``` - -### Validate backtesting results - -Once the optimized parameters and conditions have been implemented into your strategy, you should backtest the strategy to make sure everything is working as expected. - -To achieve same results (number of trades, their durations, profit, etc.) than during Hyperopt, please use same configuration and parameters (timerange, timeframe, ...) used for hyperopt `--dmmp`/`--disable-max-market-positions` and `--eps`/`--enable-position-stacking` for Backtesting. - -Should results not match, please double-check to make sure you transferred all conditions correctly. -Pay special care to the stoploss (and trailing stoploss) parameters, as these are often set in configuration files, which override changes to the strategy. -You should also carefully review the log of your backtest to ensure that there were no parameters inadvertently set by the configuration (like `stoploss` or `trailing_stop`). - -### Sharing methods with your strategy - -Hyperopt classes provide access to the Strategy via the `strategy` class attribute. -This can be a great way to reduce code duplication if used correctly, but will also complicate usage for inexperienced users. - -``` python -from pandas import DataFrame -from freqtrade.strategy.interface import IStrategy -import freqtrade.vendor.qtpylib.indicators as qtpylib - -class MyAwesomeStrategy(IStrategy): - - buy_params = { - 'rsi-value': 30, - 'adx-value': 35, - } - - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - return self.buy_strategy_generator(self.buy_params, dataframe, metadata) - - @staticmethod - def buy_strategy_generator(params, dataframe: DataFrame, metadata: dict) -> DataFrame: - dataframe.loc[ - ( - qtpylib.crossed_above(dataframe['rsi'], params['rsi-value']) & - dataframe['adx'] > params['adx-value']) & - dataframe['volume'] > 0 - ) - , 'buy'] = 1 - return dataframe - -class MyAwesomeHyperOpt(IHyperOpt): - ... - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - # Call strategy's buy strategy generator - return self.StrategyClass.buy_strategy_generator(params, dataframe, metadata) - - return populate_buy_trend -``` diff --git a/docs/bot-usage.md b/docs/bot-usage.md index b65220722..c6a7f6103 100644 --- a/docs/bot-usage.md +++ b/docs/bot-usage.md @@ -12,22 +12,22 @@ This page explains the different parameters of the bot and how to run it. ``` usage: freqtrade [-h] [-V] - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} ... Free, open source crypto trading bot positional arguments: - {trade,create-userdir,new-config,new-hyperopt,new-strategy,download-data,convert-data,convert-trade-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,plot-dataframe,plot-profit} + {trade,create-userdir,new-config,new-strategy,download-data,convert-data,convert-trade-data,list-data,backtesting,edge,hyperopt,hyperopt-list,hyperopt-show,list-exchanges,list-hyperopts,list-markets,list-pairs,list-strategies,list-timeframes,show-trades,test-pairlist,install-ui,plot-dataframe,plot-profit,webserver} trade Trade module. create-userdir Create user-data directory. new-config Create new config - new-hyperopt Create new hyperopt new-strategy Create new strategy download-data Download backtesting data. convert-data Convert candle (OHLCV) data from one format to another. convert-trade-data Convert trade data from one format to another. + list-data List downloaded data. backtesting Backtesting module. edge Edge module. hyperopt Hyperopt module. @@ -41,8 +41,10 @@ positional arguments: list-timeframes Print available timeframes for the exchange. show-trades Show trades. test-pairlist Test your pairlist configuration. + install-ui Install FreqUI plot-dataframe Plot candles with indicators. plot-profit Generate plot showing profits. + webserver Webserver module. optional arguments: -h, --help show this help message and exit diff --git a/docs/deprecated.md b/docs/deprecated.md index b7ad847e6..d86a7ac7a 100644 --- a/docs/deprecated.md +++ b/docs/deprecated.md @@ -38,3 +38,8 @@ Since only quoteVolume can be compared between assets, the other options (bidVol Using `order_book_min` and `order_book_max` used to allow stepping the orderbook and trying to find the next ROI slot - trying to place sell-orders early. As this does however increase risk and provides no benefit, it's been removed for maintainability purposes in 2021.7. + +### Legacy Hyperopt mode + +Using separate hyperopt files was deprecated in 2021.4 and was removed in 2021.9. +Please switch to the new [Parametrized Strategies](hyperopt.md) to benefit from the new hyperopt interface. diff --git a/docs/faq.md b/docs/faq.md index b8a3a44d8..285625491 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -167,7 +167,7 @@ Since hyperopt uses Bayesian search, running for too many epochs may not produce It's therefore recommended to run between 500-1000 epochs over and over until you hit at least 10.000 epochs in total (or are satisfied with the result). You can best judge by looking at the results - if the bot keeps discovering better strategies, it's best to keep on going. ```bash -freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy SampleStrategy -e 1000 +freqtrade hyperopt --hyperopt-loss SharpeHyperOptLossDaily --strategy SampleStrategy -e 1000 ``` ### Why does it take a long time to run hyperopt? diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 1eb90f1bc..e69b761c4 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -44,9 +44,8 @@ usage: freqtrade hyperopt [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--data-format-ohlcv {json,jsongz,hdf5}] [--max-open-trades INT] [--stake-amount STAKE_AMOUNT] [--fee FLOAT] - [-p PAIRS [PAIRS ...]] [--hyperopt NAME] - [--hyperopt-path PATH] [--eps] [--dmmp] - [--enable-protections] + [-p PAIRS [PAIRS ...]] [--hyperopt-path PATH] + [--eps] [--dmmp] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] [-e INT] [--spaces {all,buy,sell,roi,stoploss,trailing,protection,default} [{all,buy,sell,roi,stoploss,trailing,protection,default} ...]] [--print-all] [--no-color] [--print-json] [-j JOBS] @@ -73,10 +72,8 @@ optional arguments: -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] Limit command to these pairs. Pairs are space- separated. - --hyperopt NAME Specify hyperopt class name which will be used by the - bot. - --hyperopt-path PATH Specify additional lookup path for Hyperopt and - Hyperopt Loss functions. + --hyperopt-path PATH Specify additional lookup path for Hyperopt Loss + functions. --eps, --enable-position-stacking Allow buying the same pair multiple times (position stacking). @@ -558,7 +555,7 @@ For example, to use one month of data, pass `--timerange 20210101-20210201` (fro Full command: ```bash -freqtrade hyperopt --hyperopt --strategy --timerange 20210101-20210201 +freqtrade hyperopt --strategy --timerange 20210101-20210201 ``` ### Running Hyperopt with Smaller Search Space @@ -684,7 +681,7 @@ If you have the `generate_roi_table()` and `roi_space()` methods in your custom Override the `roi_space()` method if you need components of the ROI tables to vary in other ranges. Override the `generate_roi_table()` and `roi_space()` methods and implement your own custom approach for generation of the ROI tables during hyperoptimization if you need a different structure of the ROI tables or other amount of rows (steps). -A sample for these methods can be found in [sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +A sample for these methods can be found in the [overriding pre-defined spaces section](advanced-hyperopt.md#overriding-pre-defined-spaces). !!! Note "Reduced search space" To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. @@ -726,7 +723,7 @@ If you are optimizing stoploss values, Freqtrade creates the 'stoploss' optimiza If you have the `stoploss_space()` method in your custom hyperopt file, remove it in order to utilize Stoploss hyperoptimization space generated by Freqtrade by default. -Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +Override the `stoploss_space()` method and define the desired range in it if you need stoploss values to vary in other range during hyperoptimization. A sample for this method can be found in the [overriding pre-defined spaces section](advanced-hyperopt.md#overriding-pre-defined-spaces). !!! Note "Reduced search space" To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. @@ -764,10 +761,10 @@ As stated in the comment, you can also use it as the values of the corresponding If you are optimizing trailing stop values, Freqtrade creates the 'trailing' optimization hyperspace for you. By default, the `trailing_stop` parameter is always set to True in that hyperspace, the value of the `trailing_only_offset_is_reached` vary between True and False, the values of the `trailing_stop_positive` and `trailing_stop_positive_offset` parameters vary in the ranges 0.02...0.35 and 0.01...0.1 correspondingly, which is sufficient in most cases. -Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in [user_data/hyperopts/sample_hyperopt_advanced.py](https://github.com/freqtrade/freqtrade/blob/develop/freqtrade/templates/sample_hyperopt_advanced.py). +Override the `trailing_space()` method and define the desired range in it if you need values of the trailing stop parameters to vary in other ranges during hyperoptimization. A sample for this method can be found in the [overriding pre-defined spaces section](advanced-hyperopt.md#overriding-pre-defined-spaces). !!! Note "Reduced search space" - To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#pverriding-pre-defined-spaces) to change this to your needs. + To limit the search space further, Decimals are limited to 3 decimal places (a precision of 0.001). This is usually sufficient, every value more precise than this will usually result in overfitted results. You can however [overriding pre-defined spaces](advanced-hyperopt.md#overriding-pre-defined-spaces) to change this to your needs. ### Reproducible results diff --git a/docs/utils.md b/docs/utils.md index a1e219d6a..ade2f4047 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -26,9 +26,7 @@ optional arguments: ├── data ├── hyperopt_results ├── hyperopts -│   ├── sample_hyperopt_advanced.py │   ├── sample_hyperopt_loss.py -│   └── sample_hyperopt.py ├── notebooks │   └── strategy_analysis_example.ipynb ├── plot diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 9c6ce543f..3574dc485 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -90,7 +90,7 @@ ARGS_HYPEROPT_SHOW = ["hyperopt_list_best", "hyperopt_list_profitable", "hyperop NO_CONF_REQURIED = ["convert-data", "convert-trade-data", "download-data", "list-timeframes", "list-markets", "list-pairs", "list-strategies", "list-data", - "list-hyperopts", "hyperopt-list", "hyperopt-show", + "hyperopt-list", "hyperopt-show", "plot-dataframe", "plot-profit", "show-trades"] NO_CONF_ALLOWED = ["create-userdir", "list-exchanges", "new-strategy"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index a1790cb9a..e3c7fe464 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -1,7 +1,7 @@ """ Definition of cli arguments used in arguments.py """ -from argparse import ArgumentTypeError +from argparse import SUPPRESS, ArgumentTypeError from freqtrade import __version__, constants from freqtrade.constants import HYPEROPT_LOSS_BUILTIN @@ -203,7 +203,7 @@ AVAILABLE_CLI_OPTIONS = { # Hyperopt "hyperopt": Arg( '--hyperopt', - help='Specify hyperopt class name which will be used by the bot.', + help=SUPPRESS, metavar='NAME', required=False, ), diff --git a/freqtrade/constants.py b/freqtrade/constants.py index efcd1aaca..9ca43d459 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -69,9 +69,7 @@ DUST_PER_COIN = { # Source files with destination directories within user-directory USER_DATA_FILES = { 'sample_strategy.py': USERPATH_STRATEGIES, - 'sample_hyperopt_advanced.py': USERPATH_HYPEROPTS, 'sample_hyperopt_loss.py': USERPATH_HYPEROPTS, - 'sample_hyperopt.py': USERPATH_HYPEROPTS, 'strategy_analysis_example.ipynb': USERPATH_NOTEBOOKS, } diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 90e19782e..42a10e8d7 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -185,7 +185,7 @@ def test_exchange_resolver(default_conf, mocker, caplog): def test_validate_order_time_in_force(default_conf, mocker, caplog): caplog.set_level(logging.INFO) - # explicitly test bittrex, exchanges implementing other policies need seperate tests + # explicitly test bittrex, exchanges implementing other policies need separate tests ex = get_patched_exchange(mocker, default_conf, id="bittrex") tif = { "buy": "gtc", @@ -2464,7 +2464,7 @@ def test_fetch_order(default_conf, mocker, exchange_name, caplog): @pytest.mark.parametrize("exchange_name", EXCHANGES) def test_fetch_stoploss_order(default_conf, mocker, exchange_name): - # Don't test FTX here - that needs a seperate test + # Don't test FTX here - that needs a separate test if exchange_name == 'ftx': return default_conf['dry_run'] = True diff --git a/tests/optimize/hyperopts/hyperopt_test_sep_file.py b/tests/optimize/hyperopts/hyperopt_test_sep_file.py deleted file mode 100644 index 0fa1e1959..000000000 --- a/tests/optimize/hyperopts/hyperopt_test_sep_file.py +++ /dev/null @@ -1,202 +0,0 @@ -# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement - -from functools import reduce -from typing import Any, Callable, Dict, List - -import talib.abstract as ta -from pandas import DataFrame -from skopt.space import Categorical, Dimension, Integer - -import freqtrade.vendor.qtpylib.indicators as qtpylib -from freqtrade.optimize.hyperopt_interface import IHyperOpt - - -class HyperoptTestSepFile(IHyperOpt): - """ - Default hyperopt provided by the Freqtrade bot. - You can override it with your own Hyperopt - """ - @staticmethod - def populate_indicators(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Add several indicators needed for buy and sell strategies defined below. - """ - # ADX - dataframe['adx'] = ta.ADX(dataframe) - # MACD - macd = ta.MACD(dataframe) - dataframe['macd'] = macd['macd'] - dataframe['macdsignal'] = macd['macdsignal'] - # MFI - dataframe['mfi'] = ta.MFI(dataframe) - # RSI - dataframe['rsi'] = ta.RSI(dataframe) - # Stochastic Fast - stoch_fast = ta.STOCHF(dataframe) - dataframe['fastd'] = stoch_fast['fastd'] - # Minus-DI - dataframe['minus_di'] = ta.MINUS_DI(dataframe) - # Bollinger bands - bollinger = qtpylib.bollinger_bands(qtpylib.typical_price(dataframe), window=20, stds=2) - dataframe['bb_lowerband'] = bollinger['lower'] - dataframe['bb_upperband'] = bollinger['upper'] - # SAR - dataframe['sar'] = ta.SAR(dataframe) - - return dataframe - - @staticmethod - def buy_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the buy strategy parameters to be used by Hyperopt. - """ - def populate_buy_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Buy strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'mfi-enabled' in params and params['mfi-enabled']: - conditions.append(dataframe['mfi'] < params['mfi-value']) - if 'fastd-enabled' in params and params['fastd-enabled']: - conditions.append(dataframe['fastd'] < params['fastd-value']) - if 'adx-enabled' in params and params['adx-enabled']: - conditions.append(dataframe['adx'] > params['adx-value']) - if 'rsi-enabled' in params and params['rsi-enabled']: - conditions.append(dataframe['rsi'] < params['rsi-value']) - - # TRIGGERS - if 'trigger' in params: - if params['trigger'] == 'bb_lower': - conditions.append(dataframe['close'] < dataframe['bb_lowerband']) - if params['trigger'] == 'macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macd'], dataframe['macdsignal'] - )) - if params['trigger'] == 'sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['close'], dataframe['sar'] - )) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'buy'] = 1 - - return dataframe - - return populate_buy_trend - - @staticmethod - def indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching buy strategy parameters. - """ - return [ - Integer(10, 25, name='mfi-value'), - Integer(15, 45, name='fastd-value'), - Integer(20, 50, name='adx-value'), - Integer(20, 40, name='rsi-value'), - Categorical([True, False], name='mfi-enabled'), - Categorical([True, False], name='fastd-enabled'), - Categorical([True, False], name='adx-enabled'), - Categorical([True, False], name='rsi-enabled'), - Categorical(['bb_lower', 'macd_cross_signal', 'sar_reversal'], name='trigger') - ] - - @staticmethod - def sell_strategy_generator(params: Dict[str, Any]) -> Callable: - """ - Define the sell strategy parameters to be used by Hyperopt. - """ - def populate_sell_trend(dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Sell strategy Hyperopt will build and use. - """ - conditions = [] - - # GUARDS AND TRENDS - if 'sell-mfi-enabled' in params and params['sell-mfi-enabled']: - conditions.append(dataframe['mfi'] > params['sell-mfi-value']) - if 'sell-fastd-enabled' in params and params['sell-fastd-enabled']: - conditions.append(dataframe['fastd'] > params['sell-fastd-value']) - if 'sell-adx-enabled' in params and params['sell-adx-enabled']: - conditions.append(dataframe['adx'] < params['sell-adx-value']) - if 'sell-rsi-enabled' in params and params['sell-rsi-enabled']: - conditions.append(dataframe['rsi'] > params['sell-rsi-value']) - - # TRIGGERS - if 'sell-trigger' in params: - if params['sell-trigger'] == 'sell-bb_upper': - conditions.append(dataframe['close'] > dataframe['bb_upperband']) - if params['sell-trigger'] == 'sell-macd_cross_signal': - conditions.append(qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) - if params['sell-trigger'] == 'sell-sar_reversal': - conditions.append(qtpylib.crossed_above( - dataframe['sar'], dataframe['close'] - )) - - if conditions: - dataframe.loc[ - reduce(lambda x, y: x & y, conditions), - 'sell'] = 1 - - return dataframe - - return populate_sell_trend - - @staticmethod - def sell_indicator_space() -> List[Dimension]: - """ - Define your Hyperopt space for searching sell strategy parameters. - """ - return [ - Integer(75, 100, name='sell-mfi-value'), - Integer(50, 100, name='sell-fastd-value'), - Integer(50, 100, name='sell-adx-value'), - Integer(60, 100, name='sell-rsi-value'), - Categorical([True, False], name='sell-mfi-enabled'), - Categorical([True, False], name='sell-fastd-enabled'), - Categorical([True, False], name='sell-adx-enabled'), - Categorical([True, False], name='sell-rsi-enabled'), - Categorical(['sell-bb_upper', - 'sell-macd_cross_signal', - 'sell-sar_reversal'], name='sell-trigger') - ] - - def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. Should be a copy of same method from strategy. - Must align to populate_indicators in this file. - Only used when --spaces does not include buy space. - """ - dataframe.loc[ - ( - (dataframe['close'] < dataframe['bb_lowerband']) & - (dataframe['mfi'] < 16) & - (dataframe['adx'] > 25) & - (dataframe['rsi'] < 21) - ), - 'buy'] = 1 - - return dataframe - - def populate_sell_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: - """ - Based on TA indicators. Should be a copy of same method from strategy. - Must align to populate_indicators in this file. - Only used when --spaces does not include sell space. - """ - dataframe.loc[ - ( - (qtpylib.crossed_above( - dataframe['macdsignal'], dataframe['macd'] - )) & - (dataframe['fastd'] > 54) - ), - 'sell'] = 1 - - return dataframe diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 498b6e588..b34c3a916 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -21,8 +21,6 @@ from freqtrade.strategy.hyper import IntParameter from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) -from .hyperopts.hyperopt_test_sep_file import HyperoptTestSepFile - def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) @@ -30,7 +28,7 @@ def test_setup_hyperopt_configuration_without_arguments(mocker, default_conf, ca args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', ] config = setup_optimize_configuration(get_args(args), RunMode.HYPEROPT) @@ -62,7 +60,7 @@ def test_setup_hyperopt_configuration_with_arguments(mocker, default_conf, caplo args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--datadir', '/foo/bar', '--timeframe', '1m', '--timerange', ':100', @@ -114,7 +112,7 @@ def test_setup_hyperopt_configuration_stake_amount(mocker, default_conf) -> None args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--stake-amount', '1', '--starting-balance', '2' ] @@ -142,9 +140,7 @@ def test_start_not_installed(mocker, default_conf, import_fails) -> None: args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', - '--hyperopt-path', - str(Path(__file__).parent / "hyperopts"), + '--strategy', 'HyperoptableStrategy', '--epochs', '5', '--hyperopt-loss', 'SharpeHyperOptLossDaily', ] @@ -203,7 +199,7 @@ def test_start_filelock(mocker, hyperopt_conf, caplog) -> None: args = [ 'hyperopt', '--config', 'config.json', - '--hyperopt', 'HyperoptTestSepFile', + '--strategy', 'HyperoptableStrategy', '--hyperopt-loss', 'SharpeHyperOptLossDaily', '--epochs', '5' ] diff --git a/tests/plugins/test_pairlocks.py b/tests/plugins/test_pairlocks.py index fce3a8cd1..c694fd7c1 100644 --- a/tests/plugins/test_pairlocks.py +++ b/tests/plugins/test_pairlocks.py @@ -68,7 +68,7 @@ def test_PairLocks(use_db): # Global lock PairLocks.lock_pair('*', lock_time) assert PairLocks.is_global_lock(lock_time + timedelta(minutes=-50)) - # Global lock also locks every pair seperately + # Global lock also locks every pair separately assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50)) assert PairLocks.is_pair_locked('XRP/USDT', lock_time + timedelta(minutes=-50)) diff --git a/tests/test_directory_operations.py b/tests/test_directory_operations.py index a11200526..905b078f9 100644 --- a/tests/test_directory_operations.py +++ b/tests/test_directory_operations.py @@ -74,16 +74,12 @@ def test_copy_sample_files(mocker, default_conf, caplog) -> None: copymock = mocker.patch('shutil.copy', MagicMock()) copy_sample_files(Path('/tmp/bar')) - assert copymock.call_count == 5 + assert copymock.call_count == 3 assert copymock.call_args_list[0][0][1] == str( Path('/tmp/bar') / 'strategies/sample_strategy.py') assert copymock.call_args_list[1][0][1] == str( - Path('/tmp/bar') / 'hyperopts/sample_hyperopt_advanced.py') - assert copymock.call_args_list[2][0][1] == str( Path('/tmp/bar') / 'hyperopts/sample_hyperopt_loss.py') - assert copymock.call_args_list[3][0][1] == str( - Path('/tmp/bar') / 'hyperopts/sample_hyperopt.py') - assert copymock.call_args_list[4][0][1] == str( + assert copymock.call_args_list[2][0][1] == str( Path('/tmp/bar') / 'notebooks/strategy_analysis_example.ipynb') From 0017b3438efcf10fad1ac0224db6c7d4c6d25a0a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 11 Sep 2021 17:56:41 +0200 Subject: [PATCH 52/58] Remove list-hyperopts --- docs/utils.md | 46 ++++------------------------- freqtrade/commands/__init__.py | 6 ++-- freqtrade/commands/arguments.py | 19 ++++-------- freqtrade/commands/list_commands.py | 21 +------------ 4 files changed, 14 insertions(+), 78 deletions(-) diff --git a/docs/utils.md b/docs/utils.md index ade2f4047..d8fbcacb7 100644 --- a/docs/utils.md +++ b/docs/utils.md @@ -109,11 +109,11 @@ Using the advanced template (populates all optional functions and methods) freqtrade new-strategy --strategy AwesomeStrategy --template advanced ``` -## List Strategies and List Hyperopts +## List Strategies -Use the `list-strategies` subcommand to see all strategies in one particular directory and the `list-hyperopts` subcommand to list custom Hyperopts. +Use the `list-strategies` subcommand to see all strategies in one particular directory. -These subcommands are useful for finding problems in your environment with loading strategies or hyperopt classes: modules with strategies or hyperopt classes that contain errors and failed to load are printed in red (LOAD FAILED), while strategies or hyperopt classes with duplicate names are printed in yellow (DUPLICATE NAME). +This subcommand is useful for finding problems in your environment with loading strategies: modules with strategies that contain errors and failed to load are printed in red (LOAD FAILED), while strategies with duplicate names are printed in yellow (DUPLICATE NAME). ``` usage: freqtrade list-strategies [-h] [-v] [--logfile FILE] [-V] [-c PATH] @@ -127,34 +127,6 @@ optional arguments: --no-color Disable colorization of hyperopt results. May be useful if you are redirecting output to a file. -Common arguments: - -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). - --logfile FILE Log to the file specified. Special values are: - 'syslog', 'journald'. See the documentation for more - details. - -V, --version show program's version number and exit - -c PATH, --config PATH - Specify configuration file (default: `config.json`). - Multiple --config options may be used. Can be set to - `-` to read config from stdin. - -d PATH, --datadir PATH - Path to directory with historical backtesting data. - --userdir PATH, --user-data-dir PATH - Path to userdata directory. -``` -``` -usage: freqtrade list-hyperopts [-h] [-v] [--logfile FILE] [-V] [-c PATH] - [-d PATH] [--userdir PATH] - [--hyperopt-path PATH] [-1] [--no-color] - -optional arguments: - -h, --help show this help message and exit - --hyperopt-path PATH Specify additional lookup path for Hyperopt and - Hyperopt Loss functions. - -1, --one-column Print output in one column. - --no-color Disable colorization of hyperopt results. May be - useful if you are redirecting output to a file. - Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). --logfile FILE Log to the file specified. Special values are: @@ -174,18 +146,16 @@ Common arguments: !!! Warning Using these commands will try to load all python files from a directory. This can be a security risk if untrusted files reside in this directory, since all module-level code is executed. -Example: Search default strategies and hyperopts directories (within the default userdir). +Example: Search default strategies directories (within the default userdir). ``` bash freqtrade list-strategies -freqtrade list-hyperopts ``` -Example: Search strategies and hyperopts directory within the userdir. +Example: Search strategies directory within the userdir. ``` bash freqtrade list-strategies --userdir ~/.freqtrade/ -freqtrade list-hyperopts --userdir ~/.freqtrade/ ``` Example: Search dedicated strategy path. @@ -194,12 +164,6 @@ Example: Search dedicated strategy path. freqtrade list-strategies --strategy-path ~/.freqtrade/strategies/ ``` -Example: Search dedicated hyperopt path. - -``` bash -freqtrade list-hyperopt --hyperopt-path ~/.freqtrade/hyperopts/ -``` - ## List Exchanges Use the `list-exchanges` subcommand to see the exchanges available for the bot. diff --git a/freqtrade/commands/__init__.py b/freqtrade/commands/__init__.py index 49b184a80..a6f14cff7 100644 --- a/freqtrade/commands/__init__.py +++ b/freqtrade/commands/__init__.py @@ -13,9 +13,9 @@ from freqtrade.commands.data_commands import (start_convert_data, start_download from freqtrade.commands.deploy_commands import (start_create_userdir, start_install_ui, start_new_strategy) from freqtrade.commands.hyperopt_commands import start_hyperopt_list, start_hyperopt_show -from freqtrade.commands.list_commands import (start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, - start_list_timeframes, start_show_trades) +from freqtrade.commands.list_commands import (start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_show_trades) from freqtrade.commands.optimize_commands import start_backtesting, start_edge, start_hyperopt from freqtrade.commands.pairlist_commands import start_test_pairlist from freqtrade.commands.plot_commands import start_plot_dataframe, start_plot_profit diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 3574dc485..d424f3ce7 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -172,11 +172,11 @@ class Arguments: from freqtrade.commands import (start_backtesting, start_convert_data, start_create_userdir, start_download_data, start_edge, start_hyperopt, start_hyperopt_list, start_hyperopt_show, start_install_ui, - start_list_data, start_list_exchanges, start_list_hyperopts, - start_list_markets, start_list_strategies, - start_list_timeframes, start_new_config, start_new_strategy, - start_plot_dataframe, start_plot_profit, start_show_trades, - start_test_pairlist, start_trading, start_webserver) + start_list_data, start_list_exchanges, start_list_markets, + start_list_strategies, start_list_timeframes, + start_new_config, start_new_strategy, start_plot_dataframe, + start_plot_profit, start_show_trades, start_test_pairlist, + start_trading, start_webserver) subparsers = self.parser.add_subparsers(dest='command', # Use custom message when no subhandler is added @@ -291,15 +291,6 @@ class Arguments: list_exchanges_cmd.set_defaults(func=start_list_exchanges) self._build_args(optionlist=ARGS_LIST_EXCHANGES, parser=list_exchanges_cmd) - # Add list-hyperopts subcommand - list_hyperopts_cmd = subparsers.add_parser( - 'list-hyperopts', - help='Print available hyperopt classes.', - parents=[_common_parser], - ) - list_hyperopts_cmd.set_defaults(func=start_list_hyperopts) - self._build_args(optionlist=ARGS_LIST_HYPEROPTS, parser=list_hyperopts_cmd) - # Add list-markets subcommand list_markets_cmd = subparsers.add_parser( 'list-markets', diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 410b9b72b..38fb098a0 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -10,7 +10,7 @@ from colorama import init as colorama_init from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration -from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active, validate_exchanges @@ -92,25 +92,6 @@ def start_list_strategies(args: Dict[str, Any]) -> None: _print_objs_tabular(strategy_objs, config.get('print_colorized', False)) -def start_list_hyperopts(args: Dict[str, Any]) -> None: - """ - Print files with HyperOpt custom classes available in the directory - """ - from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver - - config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) - - directory = Path(config.get('hyperopt_path', config['user_data_dir'] / USERPATH_HYPEROPTS)) - hyperopt_objs = HyperOptResolver.search_all_objects(directory, not args['print_one_column']) - # Sort alphabetically - hyperopt_objs = sorted(hyperopt_objs, key=lambda x: x['name']) - - if args['print_one_column']: - print('\n'.join([s['name'] for s in hyperopt_objs])) - else: - _print_objs_tabular(hyperopt_objs, config.get('print_colorized', False)) - - def start_list_timeframes(args: Dict[str, Any]) -> None: """ Print timeframes available on Exchange From 236dc4800075a30a4c7ed1e52ef45be0d785ef76 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 12 Sep 2021 07:46:17 +0200 Subject: [PATCH 53/58] Update CI to use new hyperopt interface --- .github/workflows/ci.yml | 6 +++--- .travis.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb767efb1..228a60389 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,7 @@ jobs: run: | cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | @@ -180,7 +180,7 @@ jobs: run: | cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | @@ -247,7 +247,7 @@ jobs: run: | cp config_examples/config_bittrex.example.json config.json freqtrade create-userdir --userdir user_data - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily --print-all + freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily --print-all - name: Flake8 run: | diff --git a/.travis.yml b/.travis.yml index f2a6d508d..15c174bfe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ jobs: - script: - cp config_examples/config_bittrex.example.json config.json - freqtrade create-userdir --userdir user_data - - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt SampleHyperOpt --hyperopt-loss SharpeHyperOptLossDaily + - freqtrade hyperopt --datadir tests/testdata -e 5 --strategy SampleStrategy --hyperopt-loss SharpeHyperOptLossDaily name: hyperopt - script: flake8 name: flake8 From d8f48cf0e3b7376ec8047a0a22e525175cb5fc98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 03:01:14 +0000 Subject: [PATCH 54/58] Bump pandas from 1.3.2 to 1.3.3 Bumps [pandas](https://github.com/pandas-dev/pandas) from 1.3.2 to 1.3.3. - [Release notes](https://github.com/pandas-dev/pandas/releases) - [Changelog](https://github.com/pandas-dev/pandas/blob/master/RELEASE.md) - [Commits](https://github.com/pandas-dev/pandas/compare/v1.3.2...v1.3.3) --- updated-dependencies: - dependency-name: pandas dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e2bed0f9e..9e4ec97d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ numpy==1.21.2 -pandas==1.3.2 +pandas==1.3.3 ccxt==1.55.83 # Pin cryptography for now due to rust build errors with piwheels From 81039fce28dbae0fcea77a4ab952fb91a847d5aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 03:01:24 +0000 Subject: [PATCH 55/58] Bump progressbar2 from 3.53.1 to 3.53.2 Bumps [progressbar2](https://github.com/WoLpH/python-progressbar) from 3.53.1 to 3.53.2. - [Release notes](https://github.com/WoLpH/python-progressbar/releases) - [Changelog](https://github.com/WoLpH/python-progressbar/blob/develop/CHANGES.rst) - [Commits](https://github.com/WoLpH/python-progressbar/compare/v3.53.1...v3.53.2) --- updated-dependencies: - dependency-name: progressbar2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-hyperopt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-hyperopt.txt b/requirements-hyperopt.txt index d7f22634b..7dc55a9fc 100644 --- a/requirements-hyperopt.txt +++ b/requirements-hyperopt.txt @@ -8,4 +8,4 @@ scikit-optimize==0.8.1 filelock==3.0.12 joblib==1.0.1 psutil==5.8.0 -progressbar2==3.53.1 +progressbar2==3.53.2 From b13bd87625d122278644f8210d07b8cad13cd352 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 04:31:52 +0000 Subject: [PATCH 56/58] Bump ccxt from 1.55.83 to 1.56.30 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.55.83 to 1.56.30. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/exchanges.cfg) - [Commits](https://github.com/ccxt/ccxt/compare/1.55.83...1.56.30) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9e4ec97d6..aa729dd9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.21.2 pandas==1.3.3 -ccxt==1.55.83 +ccxt==1.56.30 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.8 aiohttp==3.7.4.post0 From a12c3ecc9b2c7fa005f4104f22cde5cba23829b6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Sep 2021 20:00:22 +0200 Subject: [PATCH 57/58] Remove credentials whenever dry-run is set from within the exchange --- freqtrade/configuration/__init__.py | 2 +- freqtrade/configuration/check_exchange.py | 13 ------------- freqtrade/configuration/config_setup.py | 5 ++--- freqtrade/exchange/__init__.py | 2 +- freqtrade/exchange/common.py | 13 +++++++++++++ freqtrade/exchange/exchange.py | 5 +++-- freqtrade/optimize/backtesting.py | 5 ++--- freqtrade/optimize/edge_cli.py | 6 +++--- tests/commands/test_commands.py | 2 -- tests/exchange/test_ccxt_compat.py | 2 +- tests/exchange/test_exchange.py | 19 ++++++++++++++++++- tests/test_configuration.py | 15 +-------------- 12 files changed, 45 insertions(+), 44 deletions(-) diff --git a/freqtrade/configuration/__init__.py b/freqtrade/configuration/__init__.py index 607f9cdef..730a4e47f 100644 --- a/freqtrade/configuration/__init__.py +++ b/freqtrade/configuration/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa: F401 -from freqtrade.configuration.check_exchange import check_exchange, remove_credentials +from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.config_setup import setup_utils_configuration from freqtrade.configuration.config_validation import validate_config_consistency from freqtrade.configuration.configuration import Configuration diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index c4f038103..fa1f47f9b 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -10,19 +10,6 @@ from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt, logger = logging.getLogger(__name__) -def remove_credentials(config: Dict[str, Any]) -> None: - """ - Removes exchange keys from the configuration and specifies dry-run - Used for backtesting / hyperopt / edge and utils. - Modifies the input dict! - """ - config['exchange']['key'] = '' - config['exchange']['secret'] = '' - config['exchange']['password'] = '' - config['exchange']['uid'] = '' - config['dry_run'] = True - - def check_exchange(config: Dict[str, Any], check_for_bad: bool = True) -> bool: """ Check if the exchange name in the config file is supported by Freqtrade diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 22836ab19..02f2d4089 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -3,7 +3,6 @@ from typing import Any, Dict from freqtrade.enums import RunMode -from .check_exchange import remove_credentials from .config_validation import validate_config_consistency from .configuration import Configuration @@ -21,8 +20,8 @@ def setup_utils_configuration(args: Dict[str, Any], method: RunMode) -> Dict[str configuration = Configuration(args, method) config = configuration.get_config() - # Ensure we do not use Exchange credentials - remove_credentials(config) + # Ensure these modes are using Dry-run + config['dry_run'] = True validate_config_consistency(config) return config diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index b0c88a51a..b08213d28 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -1,6 +1,6 @@ # flake8: noqa: F401 # isort: off -from freqtrade.exchange.common import MAP_EXCHANGE_CHILDCLASS +from freqtrade.exchange.common import remove_credentials, MAP_EXCHANGE_CHILDCLASS from freqtrade.exchange.exchange import Exchange # isort: on from freqtrade.exchange.bibox import Bibox diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 694aa3aa2..7b89adf06 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -51,6 +51,19 @@ EXCHANGE_HAS_OPTIONAL = [ ] +def remove_credentials(config) -> None: + """ + Removes exchange keys from the configuration and specifies dry-run + Used for backtesting / hyperopt / edge and utils. + Modifies the input dict! + """ + if config.get('dry_run', False): + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['exchange']['password'] = '' + config['exchange']['uid'] = '' + + def calculate_backoff(retrycount, max_retries): """ Calculate backoff diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 11de5411f..2b9b08d70 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -26,8 +26,8 @@ from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFun InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, - EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, - retrier_async) + EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, + remove_credentials, retrier, retrier_async) from freqtrade.misc import chunks, deep_merge_dicts, safe_value_fallback2 from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -104,6 +104,7 @@ class Exchange: # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} + remove_credentials(config) if config['dry_run']: logger.info('Instance is running with dry_run enabled') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 4b52e104b..3e06bfa1b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame -from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency +from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.constants import DATETIME_PRINT_FORMAT from freqtrade.data import history from freqtrade.data.btanalysis import trade_list_to_dataframe @@ -61,8 +61,7 @@ class Backtesting: self.config = config self.results: Optional[Dict[str, Any]] = None - # Reset keys for backtesting - remove_credentials(self.config) + config['dry_run'] = True self.strategylist: List[IStrategy] = [] self.all_results: Dict[str, Dict] = {} diff --git a/freqtrade/optimize/edge_cli.py b/freqtrade/optimize/edge_cli.py index aab7def05..417faa685 100644 --- a/freqtrade/optimize/edge_cli.py +++ b/freqtrade/optimize/edge_cli.py @@ -7,7 +7,7 @@ import logging from typing import Any, Dict from freqtrade import constants -from freqtrade.configuration import TimeRange, remove_credentials, validate_config_consistency +from freqtrade.configuration import TimeRange, validate_config_consistency from freqtrade.edge import Edge from freqtrade.optimize.optimize_reports import generate_edge_table from freqtrade.resolvers import ExchangeResolver, StrategyResolver @@ -28,8 +28,8 @@ class EdgeCli: def __init__(self, config: Dict[str, Any]) -> None: self.config = config - # Reset keys for edge - remove_credentials(self.config) + # Ensure using dry-run + self.config['dry_run'] = True self.config['stake_amount'] = constants.UNLIMITED_STAKE_AMOUNT self.exchange = ExchangeResolver.load_exchange(self.config['exchange']['name'], self.config) self.strategy = StrategyResolver.load_strategy(self.config) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1da9e5100..87040bfd5 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -32,8 +32,6 @@ def test_setup_utils_configuration(): config = setup_utils_configuration(get_args(args), RunMode.OTHER) assert "exchange" in config assert config['dry_run'] is True - assert config['exchange']['key'] == '' - assert config['exchange']['secret'] == '' def test_start_trading_fail(mocker, caplog): diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 3a32d108b..0f7963e36 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -54,7 +54,7 @@ EXCHANGES = { def exchange_conf(): config = get_default_conf((Path(__file__).parent / "testdata").resolve()) config['exchange']['pair_whitelist'] = [] - config['dry_run'] = False + # config['dry_run'] = False return config diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 90e19782e..baf1f081a 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1,5 +1,6 @@ import copy import logging +from copy import deepcopy from datetime import datetime, timedelta, timezone from math import isclose from random import randint @@ -14,7 +15,7 @@ from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOr OperationalException, PricingError, TemporaryError) from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, - calculate_backoff) + calculate_backoff, remove_credentials) from freqtrade.exchange.exchange import (market_is_active, timeframe_to_minutes, timeframe_to_msecs, timeframe_to_next_date, timeframe_to_prev_date, timeframe_to_seconds) @@ -78,6 +79,22 @@ def test_init(default_conf, mocker, caplog): assert log_has('Instance is running with dry_run enabled', caplog) +def test_remove_credentials(default_conf, caplog) -> None: + conf = deepcopy(default_conf) + conf['dry_run'] = False + remove_credentials(conf) + + assert conf['exchange']['key'] != '' + assert conf['exchange']['secret'] != '' + + conf['dry_run'] = True + remove_credentials(conf) + assert conf['exchange']['key'] == '' + assert conf['exchange']['secret'] == '' + assert conf['exchange']['password'] == '' + assert conf['exchange']['uid'] == '' + + def test_init_ccxt_kwargs(default_conf, mocker, caplog): mocker.patch('freqtrade.exchange.Exchange._load_markets', MagicMock(return_value={})) mocker.patch('freqtrade.exchange.Exchange.validate_stakecurrency') diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 9aea4fa11..1ce45e4d5 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -11,8 +11,7 @@ import pytest from jsonschema import ValidationError from freqtrade.commands import Arguments -from freqtrade.configuration import (Configuration, check_exchange, remove_credentials, - validate_config_consistency) +from freqtrade.configuration import Configuration, check_exchange, validate_config_consistency from freqtrade.configuration.config_validation import validate_config_schema from freqtrade.configuration.deprecated_settings import (check_conflicting_settings, process_deprecated_setting, @@ -617,18 +616,6 @@ def test_check_exchange(default_conf, caplog) -> None: check_exchange(default_conf) -def test_remove_credentials(default_conf, caplog) -> None: - conf = deepcopy(default_conf) - conf['dry_run'] = False - remove_credentials(conf) - - assert conf['dry_run'] is True - assert conf['exchange']['key'] == '' - assert conf['exchange']['secret'] == '' - assert conf['exchange']['password'] == '' - assert conf['exchange']['uid'] == '' - - def test_cli_verbose_with_params(default_conf, mocker, caplog) -> None: patched_configuration_load_config_file(mocker, default_conf) From 4e2b1764b8af34f67305f213ac29dcff3a59da37 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 14 Sep 2021 20:24:44 +0200 Subject: [PATCH 58/58] ccxt_compat_tests must run with dry-run=False --- tests/exchange/test_ccxt_compat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index 0f7963e36..d71dbe015 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -54,7 +54,9 @@ EXCHANGES = { def exchange_conf(): config = get_default_conf((Path(__file__).parent / "testdata").resolve()) config['exchange']['pair_whitelist'] = [] - # config['dry_run'] = False + config['exchange']['key'] = '' + config['exchange']['secret'] = '' + config['dry_run'] = False return config