From cf03daa0fd60e9a2ae2769a89bbde4cfb92d62f0 Mon Sep 17 00:00:00 2001 From: Antreas Gribas Date: Fri, 30 Apr 2021 00:28:42 +0300 Subject: [PATCH 001/127] Fix bug in running hyperopt in windows 10 with preferred encoding in locale differrent from utf-8 --- freqtrade/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index e96e7f530..30ad96b33 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,6 +1,11 @@ """ Freqtrade bot """ __version__ = 'develop' +import locale +def getpreferredencoding(do_setlocale=True): + return "utf-8" +locale.getpreferredencoding = getpreferredencoding + if __version__ == 'develop': try: From 30da307d137e997d2d0c8590fa403e3e29a2f49f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 May 2021 17:01:52 +0200 Subject: [PATCH 002/127] Remove encode/decode for hyperopt --- freqtrade/__init__.py | 5 ----- freqtrade/optimize/hyperopt.py | 3 +-- tests/optimize/test_hyperopt.py | 3 +-- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index 30ad96b33..e96e7f530 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,11 +1,6 @@ """ Freqtrade bot """ __version__ = 'develop' -import locale -def getpreferredencoding(do_setlocale=True): - return "utf-8" -locale.getpreferredencoding = getpreferredencoding - if __version__ == 'develop': try: diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index d1dabff36..6f899aba6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -4,7 +4,6 @@ This module contains the hyperopt logic """ -import locale import logging import random import warnings @@ -354,7 +353,7 @@ class Hyperopt: f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} " f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). " f"Avg duration {results_metrics['duration']:5.1f} min." - ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8') + ) def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: return Optimizer( diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 90ff8a1d0..637623e7d 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1,5 +1,4 @@ # pragma pylint: disable=missing-docstring,W0212,C0103 -import locale import logging import re from datetime import datetime @@ -631,7 +630,7 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'Avg profit 2.31%. Median profit 2.31%. Total profit ' '0.00023300 BTC ( 2.31\N{GREEK CAPITAL LETTER SIGMA}%). ' 'Avg duration 100.0 min.' - ).encode(locale.getpreferredencoding(), 'replace').decode('utf-8'), + ), 'params_details': {'buy': {'adx-enabled': False, 'adx-value': 0, 'fastd-enabled': True, From e0ca3c014c254c9e34447a6bf417d80ac1faf8a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 May 2021 17:12:48 +0200 Subject: [PATCH 003/127] Don't completely remove encode/decode --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 6f899aba6..ef842faec 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -353,7 +353,7 @@ class Hyperopt: f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} " f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). " f"Avg duration {results_metrics['duration']:5.1f} min." - ) + ).encode('utf-8', 'replace').decode('utf-8') def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: return Optimizer( From 1cb430f59b0116cbf66d71ea5f6d7909eeacea25 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 1 May 2021 17:41:40 +0200 Subject: [PATCH 004/127] Remove encoding specifics, gitattributes to echeckout as utf8 --- .gitattributes | 7 ++++--- freqtrade/optimize/hyperopt.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitattributes b/.gitattributes index 00abd1d9d..bc0382636 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ -*.py eol=lf -*.sh eol=lf -*.ps1 eol=crlf +* encoding=UTF-8 +*.py eol=lf +*.sh eol=lf +*.ps1 eol=crlf diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index ef842faec..6f899aba6 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -353,7 +353,7 @@ class Hyperopt: f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} " f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). " f"Avg duration {results_metrics['duration']:5.1f} min." - ).encode('utf-8', 'replace').decode('utf-8') + ) def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: return Optimizer( From e9841910e9f6670bcca5aa41952b806590bf91cd Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Thu, 20 May 2021 00:33:33 +0300 Subject: [PATCH 005/127] day/week options for Telegram '/profit' command --- README.md | 2 +- docs/telegram-usage.md | 4 ++-- freqtrade/rpc/rpc.py | 5 +++-- freqtrade/rpc/telegram.py | 17 +++++++++++++---- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ab9597a77..ada31b42b 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor - `/stop`: Stops the trader. - `/stopbuy`: Stop entering new trades. - `/status |[table]`: Lists all or specific open trades. -- `/profit`: Lists cumulative profit from all finished trades +- `/profit [day]|[week]`: Lists cumulative profit from all finished trades - `/forcesell |all`: Instantly sells the given trade (Ignoring `minimum_roi`). - `/performance`: Show performance of each finished trade grouped by pair - `/balance`: Show account balance per currency. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 07f5fe7dd..c477921de 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -130,7 +130,7 @@ You can create your own keyboard in `config.json`: !!! Note "Supported Commands" Only the following commands are allowed. Command arguments are not supported! - `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopbuy`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version` + `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/profit day`, `/profit week`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopbuy`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version` ## Telegram commands @@ -154,7 +154,7 @@ official commands. You can ask at any moment for help with `/help`. | `/count` | Displays number of trades used and available | `/locks` | Show currently locked pairs. | `/unlock ` | Remove the lock for this pair (or for this lock id). -| `/profit` | Display a summary of your profit/loss from close trades and some stats about your performance +| `/profit [day]|[week]` | Display a summary of your profit/loss from close trades and some stats about your performance | `/forcesell ` | Instantly sells the given trade (Ignoring `minimum_roi`). | `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`). | `/forcebuy [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 2c54d743e..7a09c1b8f 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -352,9 +352,10 @@ class RPC: return {'sell_reasons': sell_reasons, 'durations': durations} def _rpc_trade_statistics( - self, stake_currency: str, fiat_display_currency: str) -> Dict[str, Any]: + self, stake_currency: str, fiat_display_currency: str, + start_date: datetime = datetime.fromtimestamp(0)) -> Dict[str, Any]: """ Returns cumulative profit statistics """ - trades = Trade.get_trades().order_by(Trade.id).all() + trades = Trade.get_trades([Trade.open_date >= start_date]).order_by(Trade.id).all() profit_all_coin = [] profit_all_ratio = [] diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 2e288ee33..ee7493938 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,7 +5,7 @@ This module manage Telegram communication """ import json import logging -from datetime import timedelta +from datetime import datetime, timedelta from html import escape from itertools import chain from typing import Any, Callable, Dict, List, Union @@ -98,7 +98,8 @@ class Telegram(RPCHandler): # this needs refacoring of the whole telegram module (same # problem in _help()). valid_keys: List[str] = ['/start', '/stop', '/status', '/status table', - '/trades', '/profit', '/performance', '/daily', + '/trades', '/performance', '/daily', + '/profit', '/profit day', '/profit week', '/stats', '/count', '/locks', '/balance', '/stopbuy', '/reload_config', '/show_config', '/logs', '/whitelist', '/blacklist', '/edge', @@ -421,9 +422,17 @@ class Telegram(RPCHandler): stake_cur = self._config['stake_currency'] fiat_disp_cur = self._config.get('fiat_display_currency', '') + start_date = datetime.fromtimestamp(0) + if context.args: + if 'day' in context.args: + start_date = datetime.utcnow().date() + elif 'week' in context.args: + start_date = datetime.utcnow().date() - timedelta(days=7) + stats = self._rpc._rpc_trade_statistics( stake_cur, - fiat_disp_cur) + fiat_disp_cur, + start_date) profit_closed_coin = stats['profit_closed_coin'] profit_closed_percent_mean = stats['profit_closed_percent_mean'] profit_closed_percent_sum = stats['profit_closed_percent_sum'] @@ -901,7 +910,7 @@ class Telegram(RPCHandler): " `pending buy orders are marked with an asterisk (*)`\n" " `pending sell orders are marked with a double asterisk (**)`\n" "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n" - "*/profit:* `Lists cumulative profit from all finished trades`\n" + "*/profit [day]|[week]:* `Lists cumulative profit from all finished trades`\n" "*/forcesell |all:* `Instantly sells the given trade or all trades, " "regardless of profit`\n" f"{forcebuy_text if self._config.get('forcebuy_enable', False) else ''}" From 935ed36433608a2beb3dabc15274171a3eb93d29 Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Thu, 20 May 2021 01:10:22 +0300 Subject: [PATCH 006/127] day/week options for Telegram '/profit' command mypy fix --- freqtrade/rpc/telegram.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ee7493938..ebd1eacb2 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,7 +5,7 @@ This module manage Telegram communication """ import json import logging -from datetime import datetime, timedelta +from datetime import datetime, date, timedelta from html import escape from itertools import chain from typing import Any, Callable, Dict, List, Union @@ -425,9 +425,9 @@ class Telegram(RPCHandler): start_date = datetime.fromtimestamp(0) if context.args: if 'day' in context.args: - start_date = datetime.utcnow().date() + start_date = datetime.combine(date.today(), datetime.min.time()) elif 'week' in context.args: - start_date = datetime.utcnow().date() - timedelta(days=7) + start_date = datetime.combine(date.today(), datetime.min.time()) - timedelta(days=7) stats = self._rpc._rpc_trade_statistics( stake_cur, From 336f4aa6a787817d9a33ece738dc88c0a0e43af0 Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Thu, 20 May 2021 08:17:08 +0300 Subject: [PATCH 007/127] day/week options for Telegram '/profit' command isort fix --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ebd1eacb2..19c520efa 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,7 +5,7 @@ This module manage Telegram communication """ import json import logging -from datetime import datetime, date, timedelta +from datetime import date, datetime, timedelta from html import escape from itertools import chain from typing import Any, Callable, Dict, List, Union From c0d3a31ddb7ac11d1e96cb3a2ba03ecc53cd7dcf Mon Sep 17 00:00:00 2001 From: Nicolas Menescardi Date: Mon, 24 May 2021 11:08:17 -0300 Subject: [PATCH 008/127] Update strategy-advanced.md fix some typos --- docs/strategy-advanced.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/strategy-advanced.md b/docs/strategy-advanced.md index cb759eb2f..3436604a9 100644 --- a/docs/strategy-advanced.md +++ b/docs/strategy-advanced.md @@ -331,7 +331,7 @@ See [Dataframe access](#dataframe-access) for more information about dataframe u Simple, time-based order-timeouts can be configured either via strategy or in the configuration in the `unfilledtimeout` section. -However, freqtrade also offers a custom callback for both order types, which allows you to decide based on custom criteria if a order did time out or not. +However, freqtrade also offers a custom callback for both order types, which allows you to decide based on custom criteria if an order did time out or not. !!! Note Unfilled order timeouts are not relevant during backtesting or hyperopt, and are only relevant during real (live) trading. Therefore these methods are only called in these circumstances. @@ -557,7 +557,7 @@ Both attributes and methods may be overridden, altering behavior of the original ## Embedding Strategies -Freqtrade provides you with with an easy way to embed the strategy into your configuration file. +Freqtrade provides you with an easy way to embed the strategy into your configuration file. This is done by utilizing BASE64 encoding and providing this string at the strategy configuration field, in your chosen config file. From 9465fd390a36f3e1425e717506c3aed9f4302221 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 24 May 2021 17:01:53 +0200 Subject: [PATCH 009/127] Fix devcontainer --- .devcontainer/Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 19e09c969..43d54e647 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,20 +1,21 @@ FROM freqtradeorg/freqtrade:develop +USER root # Install dependencies COPY requirements-dev.txt /freqtrade/ + RUN apt-get update \ - && apt-get -y install git mercurial sudo vim \ + && apt-get -y install git mercurial sudo vim build-essential \ && apt-get clean \ - && pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \ - && useradd -u 1000 -U -m ftuser \ && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \ && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \ - && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \ - && mv /root/.local /home/ftuser/.local/ \ + && echo "export HISTFILE=~/commandhistory/.bash_hi>story" >> /home/ftuser/.bashrc \ && chown ftuser:ftuser -R /home/ftuser/.local/ \ && chown ftuser: -R /home/ftuser/ USER ftuser +RUN pip install --user autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir + # Empty the ENTRYPOINT to allow all commands ENTRYPOINT [] From 7dcf94f80cd9ce58d71bba0f515fb5f9f3b47177 Mon Sep 17 00:00:00 2001 From: Kamontat Chantrachirathumrong Date: Tue, 25 May 2021 08:18:14 +0700 Subject: [PATCH 010/127] Update Dockerfile --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 43d54e647..6389a50df 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update \ && apt-get clean \ && mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \ && echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \ - && echo "export HISTFILE=~/commandhistory/.bash_hi>story" >> /home/ftuser/.bashrc \ + && echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \ && chown ftuser:ftuser -R /home/ftuser/.local/ \ && chown ftuser: -R /home/ftuser/ From a747312c1ebaabb71e9fd2eeff85fc7c96cfb45f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 May 2021 18:02:07 +0200 Subject: [PATCH 011/127] Explicitly provide is_open to trade Object closes #5015 --- freqtrade/freqtradebot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1c3a759f4..56596f2b9 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -601,6 +601,7 @@ class FreqtradeBot(LoggingMixin): pair=pair, stake_amount=stake_amount, amount=amount, + is_open=True, amount_requested=amount_requested, fee_open=fee, fee_close=fee, From cc5769e900514aa3180feb3db90d6362996e91bd Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 May 2021 19:24:56 +0200 Subject: [PATCH 012/127] Convert np.int64 to proper int closes #5018 --- freqtrade/optimize/hyperopt.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 85bcbb8e3..430fe290a 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,6 +11,7 @@ from datetime import datetime, timezone from math import ceil from pathlib import Path from typing import Any, Dict, List, Optional +import numpy as np import progressbar import rapidjson @@ -162,8 +163,13 @@ class Hyperopt: While not a valid json object - this allows appending easily. :param epoch: result dictionary for this epoch. """ + def default_parser(x): + if isinstance(x, np.integer): + return int(x) + return str(x) + with self.results_file.open('a') as f: - rapidjson.dump(epoch, f, default=str, + rapidjson.dump(epoch, f, default=default_parser, number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN) f.write("\n") From 8e89d3e6e4aace3f8f688f19ce47b254338626de Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 25 May 2021 19:33:34 +0200 Subject: [PATCH 013/127] Fix sort error --- freqtrade/optimize/hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 430fe290a..49273d3de 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -11,8 +11,8 @@ from datetime import datetime, timezone from math import ceil from pathlib import Path from typing import Any, Dict, List, Optional -import numpy as np +import numpy as np import progressbar import rapidjson from colorama import Fore, Style From 42453333bef8f600c29007e69bd5be508953a9a8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 07:38:47 +0200 Subject: [PATCH 014/127] Align coinbase download with ccxt limits Align with https://github.com/ccxt/ccxt/issues/9268 --- freqtrade/exchange/__init__.py | 1 + freqtrade/exchange/bybit.py | 1 - freqtrade/exchange/coinbasepro.py | 23 +++++++++++++++++++++++ freqtrade/exchange/hitbtc.py | 1 - requirements.txt | 2 +- 5 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 freqtrade/exchange/coinbasepro.py diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 23ba2eb10..015e0c869 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -7,6 +7,7 @@ from freqtrade.exchange.bibox import Bibox from freqtrade.exchange.binance import Binance from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.bybit import Bybit +from freqtrade.exchange.coinbasepro import Coinbasepro from freqtrade.exchange.exchange import (available_exchanges, ccxt_exchanges, is_exchange_known_ccxt, is_exchange_officially_supported, market_is_active, timeframe_to_minutes, timeframe_to_msecs, diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py index 4a44bb42d..163f8c44e 100644 --- a/freqtrade/exchange/bybit.py +++ b/freqtrade/exchange/bybit.py @@ -18,7 +18,6 @@ class Bybit(Exchange): may still not work as expected. """ - # fetchCurrencies API point requires authentication for Bybit, _ft_has: Dict = { "ohlcv_candle_limit": 200, } diff --git a/freqtrade/exchange/coinbasepro.py b/freqtrade/exchange/coinbasepro.py new file mode 100644 index 000000000..7dd9c80dc --- /dev/null +++ b/freqtrade/exchange/coinbasepro.py @@ -0,0 +1,23 @@ +""" CoinbasePro exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Coinbasepro(Exchange): + """ + CoinbasePro exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + + Please note that this exchange is not included in the list of exchanges + officially supported by the Freqtrade development team. So some features + may still not work as expected. + """ + + _ft_has: Dict = { + "ohlcv_candle_limit": 300, + } diff --git a/freqtrade/exchange/hitbtc.py b/freqtrade/exchange/hitbtc.py index 763535263..a48c9a198 100644 --- a/freqtrade/exchange/hitbtc.py +++ b/freqtrade/exchange/hitbtc.py @@ -17,7 +17,6 @@ class Hitbtc(Exchange): may still not work as expected. """ - # fetchCurrencies API point requires authentication for Hitbtc, _ft_has: Dict = { "ohlcv_candle_limit": 1000, "ohlcv_params": {"sort": "DESC"} diff --git a/requirements.txt b/requirements.txt index df611aedf..512b2a588 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.20.3 pandas==1.2.4 -ccxt==1.50.30 +ccxt==1.50.48 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From f3d8e5c9e418d6645eb07a7bbd2e84840d656bb9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 10:44:35 +0200 Subject: [PATCH 015/127] Improve hyperopt docs closes #4949 --- docs/hyperopt.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index d8f4a8071..0d7b1cc32 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -249,15 +249,16 @@ We continue to define hyperoptable parameters: ```python class MyAwesomeStrategy(IStrategy): - buy_adx = IntParameter(20, 40, default=30, space="buy") + buy_adx = DecimalParameter(20, 40, decimals=1, default=30.1, space="buy") buy_rsi = IntParameter(20, 40, default=30, space="buy") - buy_adx_enabled = CategoricalParameter([True, False], space="buy") - buy_rsi_enabled = CategoricalParameter([True, False], space="buy") - buy_trigger = CategoricalParameter(['bb_lower', 'macd_cross_signal'], space="buy") + buy_adx_enabled = CategoricalParameter([True, False], default=True, space="buy") + buy_rsi_enabled = CategoricalParameter([True, False], default=False, space="buy") + buy_trigger = CategoricalParameter(["bb_lower", "macd_cross_signal"], default="bb_lower", space="buy") ``` -Above definition says: I have five parameters I want to randomly combine to find the best combination. -Two of them are integer values (`buy_adx` and `buy_rsi`) and I want you test in the range of values 20 to 40. +The above definition says: I have five parameters I want to randomly combine to find the best combination. +`buy_rsi` is an integer parameter, which will be tested between 20 and 40. This space has a size of 20. +`buy_adx` is a decimal parameter, which will be evaluated between 20 and 40 with 1 decimal place (so values are 20.1, 20.2, ...). This space has a size of 200. Then we have three category variables. First two are either `True` or `False`. We use these to either enable or disable the ADX and RSI guards. The last one we call `trigger` and use it to decide which buy trigger we want to use. From a6cd35365591d5ec6d7d71cec262c051470b8c03 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 11:22:22 +0200 Subject: [PATCH 016/127] Address random ci failure --- tests/rpc/test_rpc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index b005fb105..904a7e74f 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -422,7 +422,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] == '0:00:00' + assert stats['avg_duration'] in ('0:00:00', '0:00:01') assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) @@ -433,7 +433,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] == '0:00:00' + assert stats['avg_duration'] in ('0:00:00', '0:00:01') assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) assert isnan(stats['profit_all_coin']) From 8bef7217ec0af25e08b91d8cdc3829edfffd9183 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 11:24:01 +0200 Subject: [PATCH 017/127] Forgot to save :O --- tests/rpc/test_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 904a7e74f..e7a968e37 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -433,7 +433,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert stats['trade_count'] == 2 assert stats['first_trade_date'] == 'just now' assert stats['latest_trade_date'] == 'just now' - assert stats['avg_duration'] in ('0:00:00', '0:00:01') + assert stats['avg_duration'] in ('0:00:00', '0:00:01') assert stats['best_pair'] == 'ETH/BTC' assert prec_satoshi(stats['best_rate'], 6.2) assert isnan(stats['profit_all_coin']) From c5c323ca8895d45567d4d0e5a7379a6c73a51606 Mon Sep 17 00:00:00 2001 From: Kamontat Chantrachirathumrong Date: Thu, 27 May 2021 16:35:27 +0700 Subject: [PATCH 018/127] Settings notify sell in telegram base on sell reason (#5028) * BREAK: notification sell by sell reason * Update constants.py * Update telegram.py * Update telegram-usage.md * Update telegram.py * Update telegram.py * Fix test fail * Update config_full.json.example * Update telegram-usage.md * Update telegram.py * Update telegram.py * Update telegram-usage.md * validate value of sell object * Fix linter * Update constants.py * Make telegram sample slightly more positive Co-authored-by: Matthias --- config_full.json.example | 11 ++++++++++- docs/telegram-usage.md | 41 ++++++++++++++++++++++++--------------- freqtrade/constants.py | 8 +++++++- freqtrade/rpc/telegram.py | 40 +++++++++++++++++++++++++------------- 4 files changed, 69 insertions(+), 31 deletions(-) diff --git a/config_full.json.example b/config_full.json.example index 24d364fdf..6aeb756f3 100644 --- a/config_full.json.example +++ b/config_full.json.example @@ -165,7 +165,16 @@ "startup": "on", "buy": "on", "buy_fill": "on", - "sell": "on", + "sell": { + "roi": "off", + "emergency_sell": "off", + "force_sell": "off", + "sell_signal": "off", + "trailing_stop_loss": "off", + "stop_loss": "off", + "stoploss_on_exchange": "off", + "custom_sell": "off" + }, "sell_fill": "on", "buy_cancel": "on", "sell_cancel": "on" diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index 6174bf0fe..991b5f1fb 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -72,22 +72,31 @@ Example configuration showing the different settings: ``` json "telegram": { - "enabled": true, - "token": "your_telegram_token", - "chat_id": "your_telegram_chat_id", - "notification_settings": { - "status": "silent", - "warning": "on", - "startup": "off", - "buy": "silent", - "sell": "on", - "buy_cancel": "silent", - "sell_cancel": "on", - "buy_fill": "off", - "sell_fill": "off" - }, - "balance_dust_level": 0.01 - }, + "enabled": true, + "token": "your_telegram_token", + "chat_id": "your_telegram_chat_id", + "notification_settings": { + "status": "silent", + "warning": "on", + "startup": "off", + "buy": "silent", + "sell": { + "roi": "silent", + "emergency_sell": "on", + "force_sell": "on", + "sell_signal": "silent", + "trailing_stop_loss": "on", + "stop_loss": "on", + "stoploss_on_exchange": "on", + "custom_sell": "silent" + }, + "buy_cancel": "silent", + "sell_cancel": "on", + "buy_fill": "off", + "sell_fill": "off" + }, + "balance_dust_level": 0.01 +}, ``` `buy` notifications are sent when the order is placed, while `buy_fill` notifications are sent when the order is filled on the exchange. diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 5ec60eb59..2e0efc8e7 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -260,7 +260,13 @@ CONF_SCHEMA = { 'enum': TELEGRAM_SETTING_OPTIONS, 'default': 'off' }, - 'sell': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, + 'sell': { + 'type': 'object', + 'additionalProperties': { + 'type': 'string', + 'enum': TELEGRAM_SETTING_OPTIONS + } + }, 'sell_cancel': {'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS}, 'sell_fill': { 'type': 'string', diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index b9e90dc8d..cca87ad91 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -233,44 +233,58 @@ class Telegram(RPCHandler): def send_msg(self, msg: Dict[str, Any]) -> None: """ Send a message to telegram channel """ - noti = self._config['telegram'].get('notification_settings', {} - ).get(str(msg['type']), 'on') + default_noti = 'on' + + msg_type = msg['type'] + noti = '' + if msg_type == RPCMessageType.SELL: + sell_noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), {}) + # For backward compatibility sell still be string + if isinstance(noti, str): + noti = sell_noti + else: + noti = sell_noti.get(str(msg['sell_reason']), default_noti) + else: + noti = self._config['telegram'] \ + .get('notification_settings', {}).get(str(msg_type), default_noti) + if noti == 'off': - logger.info(f"Notification '{msg['type']}' not sent.") + logger.info(f"Notification '{msg_type}' not sent.") # Notification disabled return - if msg['type'] == RPCMessageType.BUY: + if msg_type == RPCMessageType.BUY: message = self._format_buy_msg(msg) - elif msg['type'] in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL): - msg['message_side'] = 'buy' if msg['type'] == RPCMessageType.BUY_CANCEL else 'sell' + elif msg_type in (RPCMessageType.BUY_CANCEL, RPCMessageType.SELL_CANCEL): + msg['message_side'] = 'buy' if msg_type == RPCMessageType.BUY_CANCEL else 'sell' message = ("\N{WARNING SIGN} *{exchange}:* " "Cancelling open {message_side} Order for {pair} (#{trade_id}). " "Reason: {reason}.".format(**msg)) - elif msg['type'] == RPCMessageType.BUY_FILL: + elif msg_type == RPCMessageType.BUY_FILL: message = ("\N{LARGE CIRCLE} *{exchange}:* " "Buy order for {pair} (#{trade_id}) filled " "for {open_rate}.".format(**msg)) - elif msg['type'] == RPCMessageType.SELL_FILL: + elif msg_type == RPCMessageType.SELL_FILL: message = ("\N{LARGE CIRCLE} *{exchange}:* " "Sell order for {pair} (#{trade_id}) filled " "for {close_rate}.".format(**msg)) - elif msg['type'] == RPCMessageType.SELL: + elif msg_type == RPCMessageType.SELL: message = self._format_sell_msg(msg) - elif msg['type'] == RPCMessageType.STATUS: + elif msg_type == RPCMessageType.STATUS: message = '*Status:* `{status}`'.format(**msg) - elif msg['type'] == RPCMessageType.WARNING: + elif msg_type == RPCMessageType.WARNING: message = '\N{WARNING SIGN} *Warning:* `{status}`'.format(**msg) - elif msg['type'] == RPCMessageType.STARTUP: + elif msg_type == RPCMessageType.STARTUP: message = '{status}'.format(**msg) else: - raise NotImplementedError('Unknown message type: {}'.format(msg['type'])) + raise NotImplementedError('Unknown message type: {}'.format(msg_type)) self._send_msg(message, disable_notification=(noti == 'silent')) From 2f79958acb3c9a120d6a82d04676266be1c12c6e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 6 Apr 2021 14:43:43 +0200 Subject: [PATCH 019/127] Move declarative_base import to import from .orm --- freqtrade/persistence/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index f2e7a10c4..6733b66a6 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -9,8 +9,7 @@ from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import Query, relationship +from sqlalchemy.orm import Query, relationship, declarative_base from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool From c31cb6711874b99f175c1508475d75ad69da4c5a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 6 Apr 2021 14:53:08 +0200 Subject: [PATCH 020/127] Further changes for sqlalchemy 1.4 --- freqtrade/persistence/migrations.py | 21 ++++++++++++--------- freqtrade/persistence/models.py | 2 +- tests/test_persistence.py | 29 ++++++++++++++++------------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index d89256baf..bb6860373 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -1,7 +1,7 @@ import logging from typing import List -from sqlalchemy import inspect +from sqlalchemy import inspect, text logger = logging.getLogger(__name__) @@ -62,15 +62,17 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col amount_requested = get_column_def(cols, 'amount_requested', 'amount') # Schema migration necessary - engine.execute(f"alter table trades rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") + with engine.begin() as connection: + connection.execute(text(f"alter table trades rename to {table_back_name}")) + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + connection.execute(text(f"drop index {index['name']}")) # let SQLAlchemy create the schema as required decl_base.metadata.create_all(engine) # Copy data back - following the correct schema - engine.execute(f"""insert into trades + with engine.begin() as connection: + connection.execute(text(f"""insert into trades (id, exchange, pair, is_open, fee_open, fee_open_cost, fee_open_currency, fee_close, fee_close_cost, fee_open_currency, open_rate, @@ -104,11 +106,12 @@ def migrate_trades_table(decl_base, inspector, engine, table_back_name: str, col {strategy} strategy, {timeframe} timeframe, {open_trade_value} open_trade_value, {close_profit_abs} close_profit_abs from {table_back_name} - """) + """)) def migrate_open_orders_to_trades(engine): - engine.execute(""" + with engine.begin() as connection: + connection.execute(text(""" insert into orders (ft_trade_id, ft_pair, order_id, ft_order_side, ft_is_open) select id ft_trade_id, pair ft_pair, open_order_id, case when close_rate_requested is null then 'buy' @@ -120,7 +123,7 @@ def migrate_open_orders_to_trades(engine): 'stoploss' ft_order_side, 1 ft_is_open from trades where stoploss_order_id is not null - """) + """)) def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List): diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 6733b66a6..6754623a8 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError -from sqlalchemy.orm import Query, relationship, declarative_base +from sqlalchemy.orm import Query, declarative_base, relationship from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlalchemy.pool import StaticPool diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 669f220bb..1814ddbbe 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock import arrow import pytest -from sqlalchemy import create_engine, inspect +from sqlalchemy import create_engine, inspect, text from freqtrade import constants from freqtrade.exceptions import DependencyException, OperationalException @@ -486,9 +486,10 @@ def test_migrate_old(mocker, default_conf, fee): mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format - engine.execute(create_table_old) - engine.execute(insert_table_old) - engine.execute(insert_table_old2) + with engine.begin() as connection: + connection.execute(text(create_table_old)) + connection.execute(text(insert_table_old)) + connection.execute(text(insert_table_old2)) # Run init to test migration init_db(default_conf['db_url'], default_conf['dry_run']) @@ -579,15 +580,16 @@ def test_migrate_new(mocker, default_conf, fee, caplog): mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format - engine.execute(create_table_old) - engine.execute("create index ix_trades_is_open on trades(is_open)") - engine.execute("create index ix_trades_pair on trades(pair)") - engine.execute(insert_table_old) + with engine.begin() as connection: + connection.execute(text(create_table_old)) + connection.execute(text("create index ix_trades_is_open on trades(is_open)")) + connection.execute(text("create index ix_trades_pair on trades(pair)")) + connection.execute(text(insert_table_old)) - # fake previous backup - engine.execute("create table trades_bak as select * from trades") + # fake previous backup + connection.execute(text("create table trades_bak as select * from trades")) - engine.execute("create table trades_bak1 as select * from trades") + connection.execute(text("create table trades_bak1 as select * from trades")) # Run init to test migration init_db(default_conf['db_url'], default_conf['dry_run']) @@ -722,8 +724,9 @@ def test_migrate_mid_state(mocker, default_conf, fee, caplog): mocker.patch('freqtrade.persistence.models.create_engine', lambda *args, **kwargs: engine) # Create table using the old format - engine.execute(create_table_old) - engine.execute(insert_table_old) + with engine.begin() as connection: + connection.execute(text(create_table_old)) + connection.execute(text(insert_table_old)) # Run init to test migration init_db(default_conf['db_url'], default_conf['dry_run']) From eaa47ff335a173d8cd01a5ec4740b470b4bf26d4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Apr 2021 17:34:20 +0200 Subject: [PATCH 021/127] Don't use autocommit --- freqtrade/persistence/models.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 6754623a8..816e20adb 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -9,9 +9,7 @@ from typing import Any, Dict, List, Optional from sqlalchemy import (Boolean, Column, DateTime, Float, ForeignKey, Integer, String, create_engine, desc, func, inspect) from sqlalchemy.exc import NoSuchModuleError -from sqlalchemy.orm import Query, declarative_base, relationship -from sqlalchemy.orm.scoping import scoped_session -from sqlalchemy.orm.session import sessionmaker +from sqlalchemy.orm import Query, declarative_base, relationship, scoped_session, sessionmaker from sqlalchemy.pool import StaticPool from sqlalchemy.sql.schema import UniqueConstraint @@ -49,7 +47,7 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None: }) try: - engine = create_engine(db_url, **kwargs) + engine = create_engine(db_url, future=True, **kwargs) except NoSuchModuleError: raise OperationalException(f"Given value for db_url: '{db_url}' " f"is no valid database URL! (See {_SQL_DOCS_URL})") @@ -57,7 +55,7 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None: # https://docs.sqlalchemy.org/en/13/orm/contextual.html#thread-local-scope # Scoped sessions proxy requests to the appropriate thread-local session. # We should use the scoped_session object - not a seperately initialized version - Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True, autocommit=True)) + Trade._session = scoped_session(sessionmaker(bind=engine, autoflush=True)) Trade.query = Trade._session.query_property() Order.query = Trade._session.query_property() PairLock.query = Trade._session.query_property() From 6fb32c35949e012ca00231e249e3c79c6fb01c07 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 13 Apr 2021 19:52:33 +0200 Subject: [PATCH 022/127] Use commit instead of .flush() --- freqtrade/freqtradebot.py | 9 ++++++--- freqtrade/persistence/models.py | 6 ++++-- freqtrade/persistence/pairlock_middleware.py | 4 ++-- freqtrade/rpc/rpc.py | 7 +++---- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 56596f2b9..c7d27b98c 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -187,7 +187,7 @@ class FreqtradeBot(LoggingMixin): if self.get_free_open_trades(): self.enter_positions() - Trade.query.session.flush() + Trade.query.session.commit() def process_stopped(self) -> None: """ @@ -620,7 +620,7 @@ class FreqtradeBot(LoggingMixin): self.update_trade_state(trade, order_id, order) Trade.query.session.add(trade) - Trade.query.session.flush() + Trade.query.session.commit() # Updating wallets self.wallets.update() @@ -706,6 +706,7 @@ class FreqtradeBot(LoggingMixin): if (self.strategy.order_types.get('stoploss_on_exchange') and self.handle_stoploss_on_exchange(trade)): trades_closed += 1 + Trade.query.session.commit() continue # Check if we can sell our current pair if trade.open_order_id is None and trade.is_open and self.handle_trade(trade): @@ -1036,6 +1037,7 @@ class FreqtradeBot(LoggingMixin): elif order['side'] == 'sell': self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) + Trade.query.session.commit() def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool: """ @@ -1233,7 +1235,7 @@ class FreqtradeBot(LoggingMixin): # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': self.update_trade_state(trade, trade.open_order_id, order) - Trade.query.session.flush() + Trade.query.session.commit() # Lock pair for one candle to prevent immediate re-buys self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), @@ -1374,6 +1376,7 @@ class FreqtradeBot(LoggingMixin): # Handling of this will happen in check_handle_timeout. return True trade.update(order) + Trade.query.session.commit() # Updating wallets when order is closed if not trade.is_open: diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 816e20adb..eaade8869 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -74,7 +74,7 @@ def cleanup_db() -> None: Flushes all pending operations to disk. :return: None """ - Trade.query.session.flush() + Trade.query.session.commit() def clean_dry_run_db() -> None: @@ -86,6 +86,7 @@ def clean_dry_run_db() -> None: # Check we are updating only a dry_run order not a prod one if 'dry_run' in trade.open_order_id: trade.open_order_id = None + Trade.query.session.commit() class Order(_DECL_BASE): @@ -174,6 +175,7 @@ class Order(_DECL_BASE): if filtered_orders: oobj = filtered_orders[0] oobj.update_from_ccxt_object(order) + Order.query.session.commit() else: logger.warning(f"Did not find order for {order}.") @@ -709,7 +711,7 @@ class Trade(_DECL_BASE, LocalTrade): Order.query.session.delete(order) Trade.query.session.delete(self) - Trade.query.session.flush() + Trade.query.session.commit() @staticmethod def get_trades_proxy(*, pair: str = None, is_open: bool = None, diff --git a/freqtrade/persistence/pairlock_middleware.py b/freqtrade/persistence/pairlock_middleware.py index 245f7cdab..af904f693 100644 --- a/freqtrade/persistence/pairlock_middleware.py +++ b/freqtrade/persistence/pairlock_middleware.py @@ -49,7 +49,7 @@ class PairLocks(): ) if PairLocks.use_db: PairLock.query.session.add(lock) - PairLock.query.session.flush() + PairLock.query.session.commit() else: PairLocks.locks.append(lock) @@ -99,7 +99,7 @@ class PairLocks(): for lock in locks: lock.active = False if PairLocks.use_db: - PairLock.query.session.flush() + PairLock.query.session.commit() @staticmethod def is_global_lock(now: Optional[datetime] = None) -> bool: diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 3f26619a9..d0755992b 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -569,7 +569,7 @@ class RPC: # Execute sell for all open orders for trade in Trade.get_open_trades(): _exec_forcesell(trade) - Trade.query.session.flush() + Trade.query.session.commit() self._freqtrade.wallets.update() return {'result': 'Created sell orders for all open trades.'} @@ -582,7 +582,7 @@ class RPC: raise RPCException('invalid argument') _exec_forcesell(trade) - Trade.query.session.flush() + Trade.query.session.commit() self._freqtrade.wallets.update() return {'result': f'Created sell order for trade {trade_id}.'} @@ -705,8 +705,7 @@ class RPC: lock.active = False lock.lock_end_time = datetime.now(timezone.utc) - # session is always the same - PairLock.query.session.flush() + PairLock.query.session.commit() return self._rpc_locks() From a01d05997e4239f0c21bcdfaaa8a24e2a7697c42 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 15 Apr 2021 07:57:52 +0200 Subject: [PATCH 023/127] Add Trade.commit method for easy use --- freqtrade/freqtradebot.py | 12 ++++++------ freqtrade/persistence/models.py | 8 ++++++-- freqtrade/rpc/rpc.py | 4 ++-- tests/test_persistence.py | 1 + 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index c7d27b98c..2e3240cfe 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -187,7 +187,7 @@ class FreqtradeBot(LoggingMixin): if self.get_free_open_trades(): self.enter_positions() - Trade.query.session.commit() + Trade.commit() def process_stopped(self) -> None: """ @@ -620,7 +620,7 @@ class FreqtradeBot(LoggingMixin): self.update_trade_state(trade, order_id, order) Trade.query.session.add(trade) - Trade.query.session.commit() + Trade.commit() # Updating wallets self.wallets.update() @@ -706,7 +706,7 @@ class FreqtradeBot(LoggingMixin): if (self.strategy.order_types.get('stoploss_on_exchange') and self.handle_stoploss_on_exchange(trade)): trades_closed += 1 - Trade.query.session.commit() + Trade.commit() continue # Check if we can sell our current pair if trade.open_order_id is None and trade.is_open and self.handle_trade(trade): @@ -1037,7 +1037,7 @@ class FreqtradeBot(LoggingMixin): elif order['side'] == 'sell': self.handle_cancel_sell(trade, order, constants.CANCEL_REASON['ALL_CANCELLED']) - Trade.query.session.commit() + Trade.commit() def handle_cancel_buy(self, trade: Trade, order: Dict, reason: str) -> bool: """ @@ -1235,7 +1235,7 @@ class FreqtradeBot(LoggingMixin): # In case of market sell orders the order can be closed immediately if order.get('status', 'unknown') == 'closed': self.update_trade_state(trade, trade.open_order_id, order) - Trade.query.session.commit() + Trade.commit() # Lock pair for one candle to prevent immediate re-buys self.strategy.lock_pair(trade.pair, datetime.now(timezone.utc), @@ -1376,7 +1376,7 @@ class FreqtradeBot(LoggingMixin): # Handling of this will happen in check_handle_timeout. return True trade.update(order) - Trade.query.session.commit() + Trade.commit() # Updating wallets when order is closed if not trade.is_open: diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index eaade8869..5648c1aee 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -74,7 +74,7 @@ def cleanup_db() -> None: Flushes all pending operations to disk. :return: None """ - Trade.query.session.commit() + Trade.commit() def clean_dry_run_db() -> None: @@ -86,7 +86,7 @@ def clean_dry_run_db() -> None: # Check we are updating only a dry_run order not a prod one if 'dry_run' in trade.open_order_id: trade.open_order_id = None - Trade.query.session.commit() + Trade.commit() class Order(_DECL_BASE): @@ -711,6 +711,10 @@ class Trade(_DECL_BASE, LocalTrade): Order.query.session.delete(order) Trade.query.session.delete(self) + Trade.commit() + + @staticmethod + def commit(): Trade.query.session.commit() @staticmethod diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index d0755992b..b55a1660a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -569,7 +569,7 @@ class RPC: # Execute sell for all open orders for trade in Trade.get_open_trades(): _exec_forcesell(trade) - Trade.query.session.commit() + Trade.commit() self._freqtrade.wallets.update() return {'result': 'Created sell orders for all open trades.'} @@ -582,7 +582,7 @@ class RPC: raise RPCException('invalid argument') _exec_forcesell(trade) - Trade.query.session.commit() + Trade.commit() self._freqtrade.wallets.update() return {'result': f'Created sell order for trade {trade_id}.'} diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 1814ddbbe..6e4030b09 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1291,6 +1291,7 @@ def test_Trade_object_idem(): excludes = ( 'delete', 'session', + 'commit', 'query', 'open_date', 'get_best_pair', From 17f74f7da8f89b0e784e9c1e81001b9374a52ebb Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 21 May 2021 14:11:02 +0200 Subject: [PATCH 024/127] Ensure commit happens on forcebuy --- freqtrade/rpc/rpc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index b55a1660a..c609bccb8 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -615,6 +615,7 @@ class RPC: # execute buy if self._freqtrade.execute_buy(pair, stakeamount, price, forcebuy=True): + Trade.commit() trade = Trade.get_trades([Trade.is_open.is_(True), Trade.pair == pair]).first() return trade else: From b82f7a2dfdeb660d1156e90b80aad68f2a8c6c5a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 May 2021 08:34:58 +0200 Subject: [PATCH 025/127] Update orders-migrations to work with new sqlalchemy syntax --- freqtrade/persistence/migrations.py | 30 +++++------ tests/test_persistence.py | 78 +++++++++++++++-------------- 2 files changed, 56 insertions(+), 52 deletions(-) diff --git a/freqtrade/persistence/migrations.py b/freqtrade/persistence/migrations.py index bb6860373..00c9b91eb 100644 --- a/freqtrade/persistence/migrations.py +++ b/freqtrade/persistence/migrations.py @@ -128,23 +128,25 @@ def migrate_open_orders_to_trades(engine): def migrate_orders_table(decl_base, inspector, engine, table_back_name: str, cols: List): # Schema migration necessary - engine.execute(f"alter table orders rename to {table_back_name}") - # drop indexes on backup table - for index in inspector.get_indexes(table_back_name): - engine.execute(f"drop index {index['name']}") + + with engine.begin() as connection: + connection.execute(text(f"alter table orders rename to {table_back_name}")) + # drop indexes on backup table + for index in inspector.get_indexes(table_back_name): + connection.execute(text(f"drop index {index['name']}")) # let SQLAlchemy create the schema as required decl_base.metadata.create_all(engine) - - engine.execute(f""" - insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, - symbol, order_type, side, price, amount, filled, average, remaining, cost, order_date, - order_filled_date, order_update_date) - select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, - symbol, order_type, side, price, amount, filled, null average, remaining, cost, order_date, - order_filled_date, order_update_date - from {table_back_name} - """) + with engine.begin() as connection: + connection.execute(text(f""" + insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, + status, symbol, order_type, side, price, amount, filled, average, remaining, cost, + order_date, order_filled_date, order_update_date) + select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, + status, symbol, order_type, side, price, amount, filled, null average, remaining, cost, + order_date, order_filled_date, order_update_date + from {table_back_name} + """)) def check_migrate(engine, decl_base, previous_tables) -> None: diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 6e4030b09..1576aaa5a 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -631,47 +631,49 @@ def test_migrate_new(mocker, default_conf, fee, caplog): caplog.clear() # Drop latest column - engine.execute("alter table orders rename to orders_bak") + with engine.begin() as connection: + connection.execute(text("alter table orders rename to orders_bak")) inspector = inspect(engine) - for index in inspector.get_indexes('orders_bak'): - engine.execute(f"drop index {index['name']}") - # Recreate table - engine.execute(""" - CREATE TABLE orders ( - id INTEGER NOT NULL, - ft_trade_id INTEGER, - ft_order_side VARCHAR NOT NULL, - ft_pair VARCHAR NOT NULL, - ft_is_open BOOLEAN NOT NULL, - order_id VARCHAR NOT NULL, - status VARCHAR, - symbol VARCHAR, - order_type VARCHAR, - side VARCHAR, - price FLOAT, - amount FLOAT, - filled FLOAT, - remaining FLOAT, - cost FLOAT, - order_date DATETIME, - order_filled_date DATETIME, - order_update_date DATETIME, - PRIMARY KEY (id), - CONSTRAINT _order_pair_order_id UNIQUE (ft_pair, order_id), - FOREIGN KEY(ft_trade_id) REFERENCES trades (id) - ) - """) + with engine.begin() as connection: + for index in inspector.get_indexes('orders_bak'): + connection.execute(text(f"drop index {index['name']}")) + # Recreate table + connection.execute(text(""" + CREATE TABLE orders ( + id INTEGER NOT NULL, + ft_trade_id INTEGER, + ft_order_side VARCHAR NOT NULL, + ft_pair VARCHAR NOT NULL, + ft_is_open BOOLEAN NOT NULL, + order_id VARCHAR NOT NULL, + status VARCHAR, + symbol VARCHAR, + order_type VARCHAR, + side VARCHAR, + price FLOAT, + amount FLOAT, + filled FLOAT, + remaining FLOAT, + cost FLOAT, + order_date DATETIME, + order_filled_date DATETIME, + order_update_date DATETIME, + PRIMARY KEY (id), + CONSTRAINT _order_pair_order_id UNIQUE (ft_pair, order_id), + FOREIGN KEY(ft_trade_id) REFERENCES trades (id) + ) + """)) - engine.execute(""" - insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, - symbol, order_type, side, price, amount, filled, remaining, cost, order_date, - order_filled_date, order_update_date) - select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, - symbol, order_type, side, price, amount, filled, remaining, cost, order_date, - order_filled_date, order_update_date - from orders_bak - """) + connection.execute(text(""" + insert into orders ( id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, + symbol, order_type, side, price, amount, filled, remaining, cost, order_date, + order_filled_date, order_update_date) + select id, ft_trade_id, ft_order_side, ft_pair, ft_is_open, order_id, status, + symbol, order_type, side, price, amount, filled, remaining, cost, order_date, + order_filled_date, order_update_date + from orders_bak + """)) # Run init to test migration init_db(default_conf['db_url'], default_conf['dry_run']) From 66de5df1d1dc1962512a20712c90ec88c6c2cb5a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 23 May 2021 08:56:41 +0200 Subject: [PATCH 026/127] Update sqlite init method --- freqtrade/persistence/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 5648c1aee..ee934f657 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -38,12 +38,14 @@ def init_db(db_url: str, clean_open_orders: bool = False) -> None: """ kwargs = {} - # Take care of thread ownership if in-memory db if db_url == 'sqlite://': kwargs.update({ - 'connect_args': {'check_same_thread': False}, 'poolclass': StaticPool, - 'echo': False, + }) + # Take care of thread ownership + if db_url.startswith('sqlite://'): + kwargs.update({ + 'connect_args': {'check_same_thread': False}, }) try: From e0083bc58e384bcd44751c4ab2eacaf90869c92c Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 13:00:05 +0200 Subject: [PATCH 027/127] Support backwards-compatible sell setting --- freqtrade/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 2e0efc8e7..e42b9d4b8 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -261,7 +261,7 @@ CONF_SCHEMA = { 'default': 'off' }, 'sell': { - 'type': 'object', + 'type': ['string', 'object'], 'additionalProperties': { 'type': 'string', 'enum': TELEGRAM_SETTING_OPTIONS From cf39dd2163f101d565a849abf545ab4897bae541 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 13:08:28 +0200 Subject: [PATCH 028/127] Fix csv-export error with new hyperopt format --- freqtrade/optimize/hyperopt_tools.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 49e70913f..38cb0854e 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -376,10 +376,11 @@ class HyperoptTools(): trials['Avg profit'] = trials['Avg profit'].apply( lambda x: f'{x * perc_multi:,.2f}%' if not isna(x) else "" ) - trials['Avg duration'] = trials['Avg duration'].apply( - lambda x: f'{x:,.1f} m' if isinstance( - x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else "" - ) + if perc_multi == 1: + trials['Avg duration'] = trials['Avg duration'].apply( + lambda x: f'{x:,.1f} m' if isinstance( + x, float) else f"{x.total_seconds() // 60:,.1f} m" if not isna(x) else "" + ) trials['Objective'] = trials['Objective'].apply( lambda x: f'{x:,.5f}' if x != 100000 else "" ) From 9fbc5c05378c84759da9d41ea39557c21df5e6fa Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 14:00:13 +0200 Subject: [PATCH 029/127] Switch to pyproject.toml for setup --- pyproject.toml | 27 ++++++++++ setup.cfg | 40 +++++++++++++++ setup.py | 130 ++++++++++++++++++------------------------------- 3 files changed, 114 insertions(+), 83 deletions(-) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..8199e446a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,27 @@ +[tool.black] +line-length = 100 +exclude = ''' +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + )/ + # Exclude vendor directory + | vendor +) +''' + +[tool.isort] +line_length = 100 + +[build-system] +requires = ["setuptools >= 46.4.0", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg index be2cd450c..26434791f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,43 @@ +[metadata] +name = freqtrade +version = attr: freqtrade.__version__ +author = Freqtrade Team +author_email = michael.egger@tsn.at +description = Freqtrade - Crypto Trading Bot +long_description = file: README.md +long_description_content_type = text/markdown +url = https://github.com/freqtrade/freqtrade +project_urls = + Bug Tracker = https://github.com/freqtrade/freqtrade/issues +license = GPLv3 +classifiers = + Environment :: Console + Intended Audience :: Science/Research + License :: OSI Approved :: GNU General Public License v3 (GPLv3) + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Operating System :: MacOS + Operating System :: Unix + Topic :: Office/Business :: Financial :: Investment + + +[options] +zip_safe = False +include_package_data = True +tests_require = + pytest + pytest-asyncio + pytest-cov + pytest-mock + +packages = find: +python_requires = >=3.6 + +[options.entry_points] +console_scripts = + freqtrade = freqtrade.main:main + [flake8] #ignore = max-line-length = 100 diff --git a/setup.py b/setup.py index 54a2e01b5..727c40c7c 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,7 @@ -from sys import version_info - from setuptools import setup -if version_info.major == 3 and version_info.minor < 7 or \ - version_info.major < 3: - print('Your Python interpreter must be 3.7 or greater!') - exit(1) - -from pathlib import Path # noqa: E402 - -from freqtrade import __version__ # noqa: E402 - - -readme_file = Path(__file__).parent / "README.md" -readme_long = "Crypto Trading Bot" -if readme_file.is_file(): - readme_long = (Path(__file__).parent / "README.md").read_text() - # Requirements used for submodules -api = ['fastapi', 'uvicorn', 'pyjwt', 'aiofiles'] plot = ['plotly>=4.0'] hyperopt = [ 'scipy', @@ -51,69 +33,51 @@ jupyter = [ 'nbconvert', ] -all_extra = api + plot + develop + jupyter + hyperopt +all_extra = plot + develop + jupyter + hyperopt -setup(name='freqtrade', - version=__version__, - description='Crypto Trading Bot', - long_description=readme_long, - long_description_content_type="text/markdown", - url='https://github.com/freqtrade/freqtrade', - author='Freqtrade Team', - author_email='michael.egger@tsn.at', - license='GPLv3', - packages=['freqtrade'], - setup_requires=['pytest-runner', 'numpy'], - tests_require=['pytest', 'pytest-asyncio', 'pytest-cov', 'pytest-mock', ], - install_requires=[ - # from requirements.txt - 'ccxt>=1.24.96', - 'SQLAlchemy', - 'python-telegram-bot>=13.4', - 'arrow>=0.17.0', - 'cachetools', - 'requests', - 'urllib3', - 'wrapt', - 'jsonschema', - 'TA-Lib', - 'technical', - 'tabulate', - 'pycoingecko', - 'py_find_1st', - 'python-rapidjson', - 'sdnotify', - 'colorama', - 'jinja2', - 'questionary', - 'prompt-toolkit', - 'numpy', - 'pandas', - 'tables', - 'blosc', - ], - extras_require={ - 'api': api, - 'dev': all_extra, - 'plot': plot, - 'jupyter': jupyter, - 'hyperopt': hyperopt, - 'all': all_extra, - }, - include_package_data=True, - zip_safe=False, - entry_points={ - 'console_scripts': [ - 'freqtrade = freqtrade.main:main', - ], - }, - classifiers=[ - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Operating System :: MacOS', - 'Operating System :: Unix', - 'Topic :: Office/Business :: Financial :: Investment', - ]) +setup( + tests_require=[ + 'pytest', + 'pytest-asyncio', + 'pytest-cov', + 'pytest-mock', + ], + install_requires=[ + # from requirements.txt + 'ccxt>=1.50.48', + 'SQLAlchemy', + 'python-telegram-bot>=13.4', + 'arrow>=0.17.0', + 'cachetools', + 'requests', + 'urllib3', + 'wrapt', + 'jsonschema', + 'TA-Lib', + 'technical', + 'tabulate', + 'pycoingecko', + 'py_find_1st', + 'python-rapidjson', + 'sdnotify', + 'colorama', + 'jinja2', + 'questionary', + 'prompt-toolkit', + 'numpy', + 'pandas', + 'tables', + 'blosc', + 'fastapi', + 'uvicorn', + 'pyjwt', + 'aiofiles' + ], + extras_require={ + 'dev': all_extra, + 'plot': plot, + 'jupyter': jupyter, + 'hyperopt': hyperopt, + 'all': all_extra, + }, +) From 3014bc34673bd8b5b32c46162d7313c0a51139ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 14:22:11 +0200 Subject: [PATCH 030/127] Don't use Sum sign in hyperopt to avoid compatibility problems --- docs/advanced-hyperopt.md | 2 +- docs/hyperopt.md | 8 ++++---- freqtrade/optimize/hyperopt.py | 2 +- tests/optimize/test_hyperopt.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/advanced-hyperopt.md b/docs/advanced-hyperopt.md index c86978b80..35fd3de4a 100644 --- a/docs/advanced-hyperopt.md +++ b/docs/advanced-hyperopt.md @@ -289,7 +289,7 @@ Given the following result from hyperopt: ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 Buy hyperspace params: { 'adx-value': 44, diff --git a/docs/hyperopt.md b/docs/hyperopt.md index b3fdc699b..1c95d84c9 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -486,7 +486,7 @@ Given the following result from hyperopt: ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 # Buy hyperspace params: buy_params = { @@ -527,7 +527,7 @@ If you are optimizing ROI (i.e. if optimization search-space contains 'all', 'de ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 # ROI table: minimal_roi = { @@ -582,7 +582,7 @@ If you are optimizing stoploss values (i.e. if optimization search-space contain ``` Best result: - 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722Σ%). Avg duration 180.4 mins. Objective: 1.94367 + 44/100: 135 trades. Avg profit 0.57%. Total profit 0.03871918 BTC (0.7722%). Avg duration 180.4 mins. Objective: 1.94367 # Buy hyperspace params: buy_params = { @@ -624,7 +624,7 @@ If you are optimizing trailing stop values (i.e. if optimization search-space co ``` Best result: - 45/100: 606 trades. Avg profit 1.04%. Total profit 0.31555614 BTC ( 630.48Σ%). Avg duration 150.3 mins. Objective: -1.10161 + 45/100: 606 trades. Avg profit 1.04%. Total profit 0.31555614 BTC ( 630.48%). Avg duration 150.3 mins. Objective: -1.10161 # Trailing stop: trailing_stop = True diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 6f899aba6..9af3710a3 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -351,7 +351,7 @@ class Hyperopt: f"Avg profit {results_metrics['avg_profit']: 6.2f}%. " f"Median profit {results_metrics['median_profit']: 6.2f}%. " f"Total profit {results_metrics['total_profit']: 11.8f} {stake_cur} " - f"({results_metrics['profit']: 7.2f}\N{GREEK CAPITAL LETTER SIGMA}%). " + f"({results_metrics['profit']: 7.2f}%). " f"Avg duration {results_metrics['duration']:5.1f} min." ) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 637623e7d..7cb04fae0 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -628,7 +628,7 @@ def test_generate_optimizer(mocker, hyperopt_conf) -> None: 'loss': 1.9840569076926293, 'results_explanation': (' 1 trades. 1/0/0 Wins/Draws/Losses. ' 'Avg profit 2.31%. Median profit 2.31%. Total profit ' - '0.00023300 BTC ( 2.31\N{GREEK CAPITAL LETTER SIGMA}%). ' + '0.00023300 BTC ( 2.31%). ' 'Avg duration 100.0 min.' ), 'params_details': {'buy': {'adx-enabled': False, From 5d961074965a5d9979a622cdf6a76a754788ca41 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 14:25:29 +0200 Subject: [PATCH 031/127] Don't configure isort twice --- pyproject.toml | 2 ++ setup.cfg | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8199e446a..f0637d8c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,8 @@ exclude = ''' [tool.isort] line_length = 100 +multi_line_output=0 +lines_after_imports=2 [build-system] requires = ["setuptools >= 46.4.0", "wheel"] diff --git a/setup.cfg b/setup.cfg index 26434791f..4882c29bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,11 +48,6 @@ exclude = .eggs, user_data, -[isort] -line_length=100 -multi_line_output=0 -lines_after_imports=2 - [mypy] ignore_missing_imports = True From 6235a4d92eff3ec4f4e0eaf5723674466be50566 Mon Sep 17 00:00:00 2001 From: Marijn Date: Thu, 27 May 2021 15:01:58 +0200 Subject: [PATCH 032/127] [changes] - Hyperopt code example --- docs/hyperopt.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/hyperopt.md b/docs/hyperopt.md index 0d7b1cc32..5f4e60b4a 100644 --- a/docs/hyperopt.md +++ b/docs/hyperopt.md @@ -237,9 +237,9 @@ class MyAwesomeStrategy(IStrategy): dataframe['macdhist'] = macd['macdhist'] bollinger = ta.BBANDS(dataframe, timeperiod=20, nbdevup=2.0, nbdevdn=2.0) - dataframe['bb_lowerband'] = boll['lowerband'] - dataframe['bb_middleband'] = boll['middleband'] - dataframe['bb_upperband'] = boll['upperband'] + dataframe['bb_lowerband'] = bollinger['lowerband'] + dataframe['bb_middleband'] = bollinger['middleband'] + dataframe['bb_upperband'] = bollinger['upperband'] return dataframe ``` From a42effd4fcc72b4033a38a934a3d8949096d666f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 May 2021 08:26:20 +0200 Subject: [PATCH 033/127] Update email to freqtrade email address --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4882c29bc..b311c94da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ name = freqtrade version = attr: freqtrade.__version__ author = Freqtrade Team -author_email = michael.egger@tsn.at +author_email = freqtrade@protonmail.com description = Freqtrade - Crypto Trading Bot long_description = file: README.md long_description_content_type = text/markdown From 8a56af919225f881916f0c008f672de9f174d967 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 May 2021 08:38:46 +0200 Subject: [PATCH 034/127] Update onlyprofit loss should use absolute profit closes #4934 --- freqtrade/optimize/hyperopt_loss_onlyprofit.py | 18 +++--------------- tests/optimize/test_hyperoptloss.py | 4 ++-- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/hyperopt_loss_onlyprofit.py b/freqtrade/optimize/hyperopt_loss_onlyprofit.py index 33f3f5bc6..4a3cf1b3b 100644 --- a/freqtrade/optimize/hyperopt_loss_onlyprofit.py +++ b/freqtrade/optimize/hyperopt_loss_onlyprofit.py @@ -9,23 +9,11 @@ from pandas import DataFrame from freqtrade.optimize.hyperopt import IHyperOptLoss -# This is assumed to be expected avg profit * expected trade count. -# For example, for 0.35% avg per trade (or 0.0035 as ratio) and 1100 trades, -# expected max profit = 3.85 -# -# Note, this is ratio. 3.85 stated above means 385Σ%, 3.0 means 300Σ%. -# -# In this implementation it's only used in calculation of the resulting value -# of the objective function as a normalization coefficient and does not -# represent any limit for profits as in the Freqtrade legacy default loss function. -EXPECTED_MAX_PROFIT = 3.0 - - class OnlyProfitHyperOptLoss(IHyperOptLoss): """ Defines the loss function for hyperopt. - This implementation takes only profit into account. + This implementation takes only absolute profit into account, not looking at any other indicator. """ @staticmethod @@ -34,5 +22,5 @@ class OnlyProfitHyperOptLoss(IHyperOptLoss): """ Objective function, returns smaller number for better results. """ - total_profit = results['profit_ratio'].sum() - return 1 - total_profit / EXPECTED_MAX_PROFIT + total_profit = results['profit_abs'].sum() + return -1 * total_profit diff --git a/tests/optimize/test_hyperoptloss.py b/tests/optimize/test_hyperoptloss.py index 73feeb007..ea0caac04 100644 --- a/tests/optimize/test_hyperoptloss.py +++ b/tests/optimize/test_hyperoptloss.py @@ -149,9 +149,9 @@ def test_sortino_loss_daily_prefers_higher_profits(default_conf, hyperopt_result def test_onlyprofit_loss_prefers_higher_profits(default_conf, hyperopt_results) -> None: results_over = hyperopt_results.copy() - results_over['profit_ratio'] = hyperopt_results['profit_ratio'] * 2 + results_over['profit_abs'] = hyperopt_results['profit_abs'] * 2 results_under = hyperopt_results.copy() - results_under['profit_ratio'] = hyperopt_results['profit_ratio'] / 2 + results_under['profit_abs'] = hyperopt_results['profit_abs'] / 2 default_conf.update({'hyperopt_loss': 'OnlyProfitHyperOptLoss'}) hl = HyperOptLossResolver.load_hyperoptloss(default_conf) From a965436cd6443f41ed0f906eed8b8a025de7e0ea Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Fri, 28 May 2021 10:17:26 +0300 Subject: [PATCH 035/127] day/week options for Telegram '/profit' command format changed to "/profit n" --- README.md | 2 +- docs/telegram-usage.md | 4 ++-- freqtrade/rpc/telegram.py | 19 ++++++++++--------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index ada31b42b..2730ee85d 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Telegram is not mandatory. However, this is a great way to control your bot. Mor - `/stop`: Stops the trader. - `/stopbuy`: Stop entering new trades. - `/status |[table]`: Lists all or specific open trades. -- `/profit [day]|[week]`: Lists cumulative profit from all finished trades +- `/profit []`: Lists cumulative profit from all finished trades, over the last n days. - `/forcesell |all`: Instantly sells the given trade (Ignoring `minimum_roi`). - `/performance`: Show performance of each finished trade grouped by pair - `/balance`: Show account balance per currency. diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index c477921de..0e6bae380 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -130,7 +130,7 @@ You can create your own keyboard in `config.json`: !!! Note "Supported Commands" Only the following commands are allowed. Command arguments are not supported! - `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/profit day`, `/profit week`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopbuy`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version` + `/start`, `/stop`, `/status`, `/status table`, `/trades`, `/profit`, `/performance`, `/daily`, `/stats`, `/count`, `/locks`, `/balance`, `/stopbuy`, `/reload_config`, `/show_config`, `/logs`, `/whitelist`, `/blacklist`, `/edge`, `/help`, `/version` ## Telegram commands @@ -154,7 +154,7 @@ official commands. You can ask at any moment for help with `/help`. | `/count` | Displays number of trades used and available | `/locks` | Show currently locked pairs. | `/unlock ` | Remove the lock for this pair (or for this lock id). -| `/profit [day]|[week]` | Display a summary of your profit/loss from close trades and some stats about your performance +| `/profit []` | Display a summary of your profit/loss from close trades and some stats about your performance, over the last n days (all trades by default) | `/forcesell ` | Instantly sells the given trade (Ignoring `minimum_roi`). | `/forcesell all` | Instantly sells all open trades (Ignoring `minimum_roi`). | `/forcebuy [rate]` | Instantly buys the given pair. Rate is optional. (`forcebuy_enable` must be set to True) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 19c520efa..4be990b96 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -206,13 +206,14 @@ class Telegram(RPCHandler): msg['emoji'] = self._get_sell_emoji(msg) message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n" + "*Profit:* `{profit_percent:.2f}%`\n" + "*Sell Reason:* `{sell_reason}`\n" + "*Duration:* `{duration} ({duration_min:.1f} min)`\n" "*Amount:* `{amount:.8f}`\n" "*Open Rate:* `{open_rate:.8f}`\n" "*Current Rate:* `{current_rate:.8f}`\n" - "*Close Rate:* `{limit:.8f}`\n" - "*Sell Reason:* `{sell_reason}`\n" - "*Duration:* `{duration} ({duration_min:.1f} min)`\n" - "*Profit:* `{profit_percent:.2f}%`").format(**msg) + "*Close Rate:* `{limit:.8f}`" + ).format(**msg) # Check if all sell properties are available. # This might not be the case if the message origin is triggered by /forcesell @@ -423,11 +424,11 @@ class Telegram(RPCHandler): fiat_disp_cur = self._config.get('fiat_display_currency', '') start_date = datetime.fromtimestamp(0) - if context.args: - if 'day' in context.args: - start_date = datetime.combine(date.today(), datetime.min.time()) - elif 'week' in context.args: - start_date = datetime.combine(date.today(), datetime.min.time()) - timedelta(days=7) + try: + timescale = int(context.args[0]) if context.args else None + start_date = datetime.combine(date.today(), datetime.min.time()) - timedelta(days=timescale) + except (TypeError, ValueError, IndexError): + pass stats = self._rpc._rpc_trade_statistics( stake_cur, From 27bd3cea4f15b19507dca486e3486ef019ffa7e0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 May 2021 12:40:30 +0200 Subject: [PATCH 036/127] Fix failing docker build --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7d5afac9c..b12cde6bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ USER ftuser # Install and execute COPY --chown=ftuser:ftuser . /freqtrade/ -RUN pip install -e . --user --no-cache-dir \ +RUN pip install -e . --user --no-cache-dir --no-build-isolation \ && mkdir /freqtrade/user_data/ \ && freqtrade install-ui From 59366208b0fdf096706f6dd599fe5e9635445aaf Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 28 May 2021 13:01:09 +0200 Subject: [PATCH 037/127] Add no_build-isolation to arm images too --- Dockerfile.armhf | 2 +- docker/Dockerfile.aarch64 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.armhf b/Dockerfile.armhf index 2b3bca042..9b7986240 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -49,7 +49,7 @@ USER ftuser # Install and execute COPY --chown=ftuser:ftuser . /freqtrade/ -RUN pip install -e . --user --no-cache-dir \ +RUN pip install -e . --user --no-cache-dir --no-build-isolation\ && mkdir /freqtrade/user_data/ \ && freqtrade install-ui diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 index 71c10d949..9f51ba61e 100644 --- a/docker/Dockerfile.aarch64 +++ b/docker/Dockerfile.aarch64 @@ -49,7 +49,7 @@ USER ftuser # Install and execute COPY --chown=ftuser:ftuser . /freqtrade/ -RUN pip install -e . --user --no-cache-dir \ +RUN pip install -e . --user --no-cache-dir --no-build-isolation\ && mkdir /freqtrade/user_data/ \ && freqtrade install-ui From 4b5a9d8c497f2a093cd226dcdf9835de2e9550f2 Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Fri, 28 May 2021 14:43:57 +0300 Subject: [PATCH 038/127] day/week options for Telegram '/profit' command revert accidental changes --- freqtrade/rpc/telegram.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 4be990b96..ef13d25f0 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -206,14 +206,13 @@ class Telegram(RPCHandler): msg['emoji'] = self._get_sell_emoji(msg) message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n" - "*Profit:* `{profit_percent:.2f}%`\n" - "*Sell Reason:* `{sell_reason}`\n" - "*Duration:* `{duration} ({duration_min:.1f} min)`\n" "*Amount:* `{amount:.8f}`\n" "*Open Rate:* `{open_rate:.8f}`\n" "*Current Rate:* `{current_rate:.8f}`\n" - "*Close Rate:* `{limit:.8f}`" - ).format(**msg) + "*Close Rate:* `{limit:.8f}`\n" + "*Sell Reason:* `{sell_reason}`\n" + "*Duration:* `{duration} ({duration_min:.1f} min)`\n" + "*Profit:* `{profit_percent:.2f}%`").format(**msg) # Check if all sell properties are available. # This might not be the case if the message origin is triggered by /forcesell From 36b68d3702e9497cd1a7bd36779d0ad46676f6dd Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Fri, 28 May 2021 14:46:22 +0300 Subject: [PATCH 039/127] day/week options for Telegram '/profit' command format changed to "/profit n" --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ef13d25f0..27dea30fd 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -910,7 +910,7 @@ class Telegram(RPCHandler): " `pending buy orders are marked with an asterisk (*)`\n" " `pending sell orders are marked with a double asterisk (**)`\n" "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n" - "*/profit [day]|[week]:* `Lists cumulative profit from all finished trades`\n" + "*/profit []:* `Lists cumulative profit from all finished trades, over the last n days`\n" "*/forcesell |all:* `Instantly sells the given trade or all trades, " "regardless of profit`\n" f"{forcebuy_text if self._config.get('forcebuy_enable', False) else ''}" From 012309a06a84d556b1cbece295b92a2585c9585e Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Fri, 28 May 2021 17:03:31 +0300 Subject: [PATCH 040/127] day/week options for Telegram '/profit' command fixed line lenght --- freqtrade/rpc/telegram.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 27dea30fd..b86f1b29c 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -425,7 +425,8 @@ class Telegram(RPCHandler): start_date = datetime.fromtimestamp(0) try: timescale = int(context.args[0]) if context.args else None - start_date = datetime.combine(date.today(), datetime.min.time()) - timedelta(days=timescale) + today_start = datetime.combine(date.today(), datetime.min.time()) + start_date = today_start - timedelta(days=timescale) except (TypeError, ValueError, IndexError): pass @@ -910,7 +911,8 @@ class Telegram(RPCHandler): " `pending buy orders are marked with an asterisk (*)`\n" " `pending sell orders are marked with a double asterisk (**)`\n" "*/trades [limit]:* `Lists last closed trades (limited to 10 by default)`\n" - "*/profit []:* `Lists cumulative profit from all finished trades, over the last n days`\n" + "*/profit []:* `Lists cumulative profit from all finished trades, " + "over the last n days`\n" "*/forcesell |all:* `Instantly sells the given trade or all trades, " "regardless of profit`\n" f"{forcebuy_text if self._config.get('forcebuy_enable', False) else ''}" From 14df243661f817fb7ade4155cbb91e97e3e89cc6 Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Fri, 28 May 2021 17:18:23 +0300 Subject: [PATCH 041/127] day/week options for Telegram '/profit' command mypy fix --- freqtrade/rpc/telegram.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index b86f1b29c..db709c556 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -424,9 +424,10 @@ class Telegram(RPCHandler): start_date = datetime.fromtimestamp(0) try: - timescale = int(context.args[0]) if context.args else None - today_start = datetime.combine(date.today(), datetime.min.time()) - start_date = today_start - timedelta(days=timescale) + if context.args: + timescale = int(context.args[0]) + today_start = datetime.combine(date.today(), datetime.min.time()) + start_date = today_start - timedelta(days=timescale) except (TypeError, ValueError, IndexError): pass From 4617967e14fc01e82c5dc14ea3e37fddf8540963 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 26 May 2021 21:47:40 +0200 Subject: [PATCH 042/127] Try building for multiarch --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a0837eb2..ab640789e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,7 @@ on: - master - stable - develop + - test_multiarch tags: release: types: [published] @@ -402,7 +403,7 @@ jobs: - name: Build Raspberry docker image env: IMAGE_NAME: freqtradeorg/freqtrade - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}_pi + BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} run: | build_helpers/publish_docker_pi.sh From 6418f2eedb154854039e3613894b304f92131ab0 Mon Sep 17 00:00:00 2001 From: Jyothish Kumar M S Date: Sat, 29 May 2021 01:28:20 +0530 Subject: [PATCH 043/127] Removed binance.je from exchange specific notes Binance Jersey is deprecated, so I think it should be removed from freqtrade --- docs/exchanges.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 8797ade8c..4a60c7683 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -14,11 +14,10 @@ Accounts having BNB accounts use this to pay for fees - if your first trade happ ### Binance sites -Binance has been split into 3, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. +Binance has been split into 2, and users must use the correct ccxt exchange ID for their exchange, otherwise API keys are not recognized. * [binance.com](https://www.binance.com/) - International users. Use exchange id: `binance`. * [binance.us](https://www.binance.us/) - US based users. Use exchange id: `binanceus`. -* [binance.je](https://www.binance.je/) - Binance Jersey, trading fiat currencies. Use exchange id: `binanceje`. ## Kraken From 313567d07d0616bd40bb5b785dbf61eeb8d40aea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 May 2021 08:12:25 +0200 Subject: [PATCH 044/127] Support having numbers in custom keyboard --- freqtrade/rpc/telegram.py | 22 +++++++++++++--------- tests/rpc/test_rpc_telegram.py | 4 ++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index db709c556..6c7fa0493 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -5,6 +5,7 @@ This module manage Telegram communication """ import json import logging +import re from datetime import date, datetime, timedelta from html import escape from itertools import chain @@ -97,24 +98,27 @@ class Telegram(RPCHandler): # TODO: DRY! - its not good to list all valid cmds here. But otherwise # this needs refacoring of the whole telegram module (same # problem in _help()). - valid_keys: List[str] = ['/start', '/stop', '/status', '/status table', - '/trades', '/performance', '/daily', - '/profit', '/profit day', '/profit week', - '/stats', '/count', '/locks', '/balance', - '/stopbuy', '/reload_config', '/show_config', - '/logs', '/whitelist', '/blacklist', '/edge', - '/help', '/version'] + valid_keys: List[str] = [r'/start$', r'/stop$', r'/status$', r'/status table$', + r'/trades$', r'/performance$', r'/daily$', r'/daily \d+$', + r'/profit$', r'/profit \d+', + r'/stats$', r'/count$', r'/locks$', r'/balance$', + r'/stopbuy$', r'/reload_config$', r'/show_config$', + r'/logs$', r'/whitelist$', r'/blacklist$', r'/edge$', + r'/forcebuy$', r'/help$', r'/version$'] + # Create keys for generation + valid_keys_print = [k.replace('$', '') for k in valid_keys] # custom keyboard specified in config.json cust_keyboard = self._config['telegram'].get('keyboard', []) if cust_keyboard: + combined = "(" + ")|(".join(valid_keys) + ")" # check for valid shortcuts invalid_keys = [b for b in chain.from_iterable(cust_keyboard) - if b not in valid_keys] + if not re.match(combined, b)] if len(invalid_keys): err_msg = ('config.telegram.keyboard: Invalid commands for ' f'custom Telegram keyboard: {invalid_keys}' - f'\nvalid commands are: {valid_keys}') + f'\nvalid commands are: {valid_keys_print}') raise OperationalException(err_msg) else: self._keyboard = cust_keyboard diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 6008ede66..a9af498b7 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1568,7 +1568,7 @@ def test__send_msg_keyboard(default_conf, mocker, caplog) -> None: ['/count', '/start', '/stop', '/help']] default_keyboard = ReplyKeyboardMarkup(default_keys_list) - custom_keys_list = [['/daily', '/stats', '/balance', '/profit'], + custom_keys_list = [['/daily', '/stats', '/balance', '/profit', '/profit 5'], ['/count', '/start', '/reload_config', '/help']] custom_keyboard = ReplyKeyboardMarkup(custom_keys_list) @@ -1602,5 +1602,5 @@ def test__send_msg_keyboard(default_conf, mocker, caplog) -> None: used_keyboard = bot.send_message.call_args[1]['reply_markup'] assert used_keyboard == custom_keyboard assert log_has("using custom keyboard from config.json: " - "[['/daily', '/stats', '/balance', '/profit'], ['/count', " + "[['/daily', '/stats', '/balance', '/profit', '/profit 5'], ['/count', " "'/start', '/reload_config', '/help']]", caplog) From 8658be004e00eb0c75fe364407ed54d2d8dd46bf Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 06:35:41 +0200 Subject: [PATCH 045/127] Use docker-manifest to build multiarch images --- build_helpers/publish_docker_pi.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh index 060b1deaf..d3a941a1f 100755 --- a/build_helpers/publish_docker_pi.sh +++ b/build_helpers/publish_docker_pi.sh @@ -3,7 +3,9 @@ # The below assumes a correctly setup docker buildx environment # Replace / with _ to create a valid tag -TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") +TAG_ORIG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") +TAG="${TAG_ORIG}_pi" + PI_PLATFORM="linux/arm/v7" echo "Running for ${TAG}" CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache @@ -30,6 +32,13 @@ else -t ${IMAGE_NAME}:${TAG} --push . fi +docker images + +docker manifest create freqtradeorg/freqtrade:${TAG}_multi ${IMAGE_NAME}:${TAG_ORIG} ${IMAGE_NAME}:${TAG} +docker manifest push freqtradeorg/freqtrade:${TAG}_multi + +docker images + if [ $? -ne 0 ]; then echo "failed building image" return 1 From 1e052bde90d8a511f3247b40733b4a5b1e337452 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 10:23:08 +0200 Subject: [PATCH 046/127] Move Dockerfile.armhf to docker directory --- .dockerignore | 1 + .github/workflows/ci.yml | 14 +++++++------- build_helpers/publish_docker.sh | 1 + build_helpers/publish_docker_pi.sh | 10 +++++----- Dockerfile.armhf => docker/Dockerfile.armhf | 0 5 files changed, 14 insertions(+), 12 deletions(-) rename Dockerfile.armhf => docker/Dockerfile.armhf (100%) diff --git a/.dockerignore b/.dockerignore index 889a4dfc7..abc5b82f0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ Dockerfile Dockerfile.armhf .dockerignore +docker/ .coveragerc .eggs .github diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab640789e..dbcde8adf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -375,13 +375,6 @@ jobs: run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Build and test and push docker image - env: - IMAGE_NAME: freqtradeorg/freqtrade - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} - run: | - build_helpers/publish_docker.sh - # We need docker experimental to pull the ARM image. - name: Switch docker to experimental run: | @@ -400,6 +393,13 @@ jobs: - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} + - name: Build and test and push docker image + env: + IMAGE_NAME: freqtradeorg/freqtrade + BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} + run: | + build_helpers/publish_docker.sh + - name: Build Raspberry docker image env: IMAGE_NAME: freqtradeorg/freqtrade diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker.sh index d987bcc69..da9fc4e34 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker.sh @@ -53,6 +53,7 @@ docker images docker push ${IMAGE_NAME} docker push ${IMAGE_NAME}:$TAG_PLOT docker push ${IMAGE_NAME}:$TAG + if [ $? -ne 0 ]; then echo "failed pushing repo" return 1 diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh index d3a941a1f..7c77c5ba1 100755 --- a/build_helpers/publish_docker_pi.sh +++ b/build_helpers/publish_docker_pi.sh @@ -7,7 +7,7 @@ TAG_ORIG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") TAG="${TAG_ORIG}_pi" PI_PLATFORM="linux/arm/v7" -echo "Running for ${TAG}" +echo "Running for ${TAG_ORIG}" CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache # Add commit and commit_message to docker container @@ -17,7 +17,7 @@ if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" docker buildx build \ --cache-to=type=registry,ref=${CACHE_TAG} \ - -f Dockerfile.armhf \ + -f docker/Dockerfile.armhf \ --platform ${PI_PLATFORM} \ -t ${IMAGE_NAME}:${TAG} --push . else @@ -27,15 +27,15 @@ else docker buildx build \ --cache-from=type=registry,ref=${CACHE_TAG} \ --cache-to=type=registry,ref=${CACHE_TAG} \ - -f Dockerfile.armhf \ + -f docker/Dockerfile.armhf \ --platform ${PI_PLATFORM} \ -t ${IMAGE_NAME}:${TAG} --push . fi docker images -docker manifest create freqtradeorg/freqtrade:${TAG}_multi ${IMAGE_NAME}:${TAG_ORIG} ${IMAGE_NAME}:${TAG} -docker manifest push freqtradeorg/freqtrade:${TAG}_multi +docker manifest create freqtradeorg/freqtrade:${TAG_ORIG} ${IMAGE_NAME}:${TAG_ORIG} ${IMAGE_NAME}:${TAG} +docker manifest push freqtradeorg/freqtrade:${TAG_ORIG} docker images diff --git a/Dockerfile.armhf b/docker/Dockerfile.armhf similarity index 100% rename from Dockerfile.armhf rename to docker/Dockerfile.armhf From 9cf2c2201b61a2ddbdb47aa622d0c1b4cfc74dd6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 27 May 2021 15:51:16 +0200 Subject: [PATCH 047/127] Align dockerfiles --- Dockerfile | 12 ++++++------ build_helpers/publish_docker_pi.sh | 6 +++++- docker/Dockerfile.aarch64 | 6 +++--- docker/Dockerfile.armhf | 5 +++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index b12cde6bb..f2d7c8a40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,8 @@ ENV FT_APP_ENV="docker" # Prepare environment RUN mkdir /freqtrade \ - && apt update \ - && apt install -y sudo \ + && apt-get update \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ && apt-get clean \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ @@ -22,10 +22,10 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev git \ - && apt-get clean \ - && pip install --upgrade pip +RUN apt-get update \ + && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \ + && apt-get clean \ + && pip install --upgrade pip # Install TA-lib COPY build_helpers/* /tmp/ diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh index 7c77c5ba1..c7024828b 100755 --- a/build_helpers/publish_docker_pi.sh +++ b/build_helpers/publish_docker_pi.sh @@ -7,7 +7,7 @@ TAG_ORIG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") TAG="${TAG_ORIG}_pi" PI_PLATFORM="linux/arm/v7" -echo "Running for ${TAG_ORIG}" +echo "Running for ${TAG}" CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache # Add commit and commit_message to docker container @@ -34,6 +34,10 @@ fi docker images +# Create multiarch image +# Make sure that all images contained here are pushed to github first. +# Otherwise installation might fail. + docker manifest create freqtradeorg/freqtrade:${TAG_ORIG} ${IMAGE_NAME}:${TAG_ORIG} ${IMAGE_NAME}:${TAG} docker manifest push freqtradeorg/freqtrade:${TAG_ORIG} diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 index 9f51ba61e..e5d3f0ee9 100644 --- a/docker/Dockerfile.aarch64 +++ b/docker/Dockerfile.aarch64 @@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker" # Prepare environment RUN mkdir /freqtrade \ && apt-get update \ - && apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ && apt-get clean \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ @@ -22,8 +22,8 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps -RUN apt-get update \ - && apt-get -y install curl build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \ +RUN apt-get update \ + && apt-get -y install build-essential libssl-dev git libffi-dev libgfortran5 pkg-config cmake gcc \ && apt-get clean \ && pip install --upgrade pip diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index 9b7986240..8abf0e44b 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker" # Prepare environment RUN mkdir /freqtrade \ && apt-get update \ - && apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ && apt-get clean \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ @@ -22,7 +22,8 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps -RUN apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \ +RUN apt-get update \ + && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \ && apt-get clean \ && pip install --upgrade pip \ && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf From f6b1abe23fe40fe6f51659b33894447c8eddf7cd Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 May 2021 08:22:50 +0200 Subject: [PATCH 048/127] Remove ci from test_multiarch again --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbcde8adf..ea766d77d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,6 @@ on: - master - stable - develop - - test_multiarch tags: release: types: [published] From d7fdc2114ac6b9c5b4fd9d1eaa0807a59ef169f4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 29 May 2021 13:02:18 +0200 Subject: [PATCH 049/127] allow list-strategies to show if params are hyperoptable --- freqtrade/commands/list_commands.py | 17 ++++++++++++++--- freqtrade/strategy/hyper.py | 22 ++++++++++++++++++---- tests/strategy/test_interface.py | 9 +++++++-- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index fa4bc1066..167847a0d 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -54,15 +54,21 @@ def _print_objs_tabular(objs: List, print_colorized: bool) -> None: reset = '' names = [s['name'] for s in objs] - objss_to_print = [{ + objs_to_print = [{ 'name': s['name'] if s['name'] else "--", 'location': s['location'].name, 'status': (red + "LOAD FAILED" + reset if s['class'] is None else "OK" if names.count(s['name']) == 1 else yellow + "DUPLICATE NAME" + reset) } for s in objs] - - print(tabulate(objss_to_print, headers='keys', tablefmt='psql', stralign='right')) + for idx, s in enumerate(objs): + if 'hyperoptable' in s: + objs_to_print[idx].update({ + 'hyperoptable': "Yes" if s['hyperoptable']['count'] > 0 else "No", + 'buy-Params': len(s['hyperoptable'].get('buy', [])), + 'sell-Params': len(s['hyperoptable'].get('sell', [])), + }) + print(tabulate(objs_to_print, headers='keys', tablefmt='psql', stralign='right')) def start_list_strategies(args: Dict[str, Any]) -> None: @@ -75,6 +81,11 @@ def start_list_strategies(args: Dict[str, Any]) -> None: strategy_objs = StrategyResolver.search_all_objects(directory, not args['print_one_column']) # Sort alphabetically strategy_objs = sorted(strategy_objs, key=lambda x: x['name']) + for obj in strategy_objs: + if obj['class']: + obj['hyperoptable'] = obj['class'].detect_all_parameters() + else: + obj['hyperoptable'] = {'count': 0} if args['print_one_column']: print('\n'.join([s['name'] for s in strategy_objs])) diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index 7dee47d87..a320f2bb8 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -273,11 +273,12 @@ class HyperStrategyMixin(object): for par in params: yield par.name, par - def _detect_parameters(self, category: str) -> Iterator[Tuple[str, BaseParameter]]: + @classmethod + def detect_parameters(cls, category: str) -> Iterator[Tuple[str, BaseParameter]]: """ Detect all parameters for 'category' """ - for attr_name in dir(self): + for attr_name in dir(cls): if not attr_name.startswith('__'): # Ignore internals, not strictly necessary. - attr = getattr(self, attr_name) + attr = getattr(cls, attr_name) if issubclass(attr.__class__, BaseParameter): if (attr_name.startswith(category + '_') and attr.category is not None and attr.category != category): @@ -287,6 +288,19 @@ class HyperStrategyMixin(object): (attr_name.startswith(category + '_') and attr.category is None)): yield attr_name, attr + @classmethod + def detect_all_parameters(cls) -> Dict: + """ Detect all parameters and return them as a list""" + params: Dict = { + 'buy': list(cls.detect_parameters('buy')), + 'sell': list(cls.detect_parameters('sell')), + } + params.update({ + 'count': len(params['buy'] + params['sell']) + }) + + return params + def _load_hyper_params(self, hyperopt: bool = False) -> None: """ Load Hyperoptable parameters @@ -303,7 +317,7 @@ class HyperStrategyMixin(object): logger.info(f"No params for {space} found, using default values.") param_container: List[BaseParameter] = getattr(self, f"ft_{space}_params") - for attr_name, attr in self._detect_parameters(space): + for attr_name, attr in self.detect_parameters(space): attr.name = attr_name attr.in_space = hyperopt and HyperoptTools.has_space(self.config, space) if not attr.category: diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index ded396779..df487986f 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -667,8 +667,13 @@ def test_auto_hyperopt_interface(default_conf): # Parameter is disabled - so value from sell_param dict will NOT be used. assert strategy.sell_minusdi.value == 0.5 + all_params = strategy.detect_all_parameters() + assert isinstance(all_params, dict) + assert len(all_params['buy']) == 2 + assert len(all_params['sell']) == 2 + assert all_params['count'] == 4 - strategy.sell_rsi = IntParameter([0, 10], default=5, space='buy') + strategy.__class__.sell_rsi = IntParameter([0, 10], default=5, space='buy') with pytest.raises(OperationalException, match=r"Inconclusive parameter.*"): - [x for x in strategy._detect_parameters('sell')] + [x for x in strategy.detect_parameters('sell')] From 08f96df3ac94bc99398d84a225236b649648e7e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 May 2021 07:24:03 +0200 Subject: [PATCH 050/127] Don't write to testdir, but to tempdir --- tests/commands/test_commands.py | 14 ++++--- tests/data/test_converter.py | 49 +++++++++++++------------ tests/optimize/test_optimize_reports.py | 8 ++-- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 4d3937d87..f47fc46c1 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -919,7 +919,8 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, - saved_hyperopt_results_legacy): + saved_hyperopt_results_legacy, tmpdir): + csv_file = Path(tmpdir) / "test.csv" for _ in (saved_hyperopt_results, saved_hyperopt_results_legacy): mocker.patch( 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', @@ -1139,17 +1140,18 @@ def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, "hyperopt-list", "--no-details", "--no-color", - "--export-csv", "test_file.csv", + "--export-csv", + str(csv_file), ] pargs = get_args(args) pargs['config'] = None start_hyperopt_list(pargs) captured = capsys.readouterr() log_has("CSV file created: test_file.csv", caplog) - f = Path("test_file.csv") - assert 'Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' in f.read_text() - assert f.is_file() - f.unlink() + assert ('Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' + in csv_file.read_text()) + assert csv_file.is_file() + csv_file.unlink() def test_hyperopt_show(mocker, capsys, saved_hyperopt_results): diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 68960af1c..31ce7255e 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -1,5 +1,7 @@ # pragma pylint: disable=missing-docstring, C0103 import logging +from pathlib import Path +from shutil import copyfile import pytest @@ -11,7 +13,7 @@ from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_forma from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) from tests.conftest import log_has, log_has_re -from tests.data.test_history import _backup_file, _clean_test_file +from tests.data.test_history import _clean_test_file def test_dataframe_correct_columns(result): @@ -251,17 +253,19 @@ def test_trades_dict_to_list(fetch_trades_result): assert t[6] == fetch_trades_result[i]['cost'] -def test_convert_trades_format(mocker, default_conf, testdatadir): - files = [{'old': testdatadir / "XRP_ETH-trades.json.gz", - 'new': testdatadir / "XRP_ETH-trades.json"}, - {'old': testdatadir / "XRP_OLD-trades.json.gz", - 'new': testdatadir / "XRP_OLD-trades.json"}, +def test_convert_trades_format(default_conf, testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) + files = [{'old': tmpdir1 / "XRP_ETH-trades.json.gz", + 'new': tmpdir1 / "XRP_ETH-trades.json"}, + {'old': tmpdir1 / "XRP_OLD-trades.json.gz", + 'new': tmpdir1 / "XRP_OLD-trades.json"}, ] for file in files: - _backup_file(file['old'], copy_file=True) + copyfile(testdatadir / file['old'].name, file['old']) + # _backup_file(file['old'], copy_file=True) assert not file['new'].exists() - default_conf['datadir'] = testdatadir + default_conf['datadir'] = tmpdir1 convert_trades_format(default_conf, convert_from='jsongz', convert_to='json', erase=False) @@ -284,14 +288,20 @@ def test_convert_trades_format(mocker, default_conf, testdatadir): file['new'].unlink() -def test_convert_ohlcv_format(mocker, default_conf, testdatadir): - file1 = testdatadir / "XRP_ETH-5m.json" - file1_new = testdatadir / "XRP_ETH-5m.json.gz" - file2 = testdatadir / "XRP_ETH-1m.json" - file2_new = testdatadir / "XRP_ETH-1m.json.gz" - _backup_file(file1, copy_file=True) - _backup_file(file2, copy_file=True) - default_conf['datadir'] = testdatadir +def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) + + file1_orig = testdatadir / "XRP_ETH-5m.json" + file1 = tmpdir1 / "XRP_ETH-5m.json" + file1_new = tmpdir1 / "XRP_ETH-5m.json.gz" + file2_orig = testdatadir / "XRP_ETH-1m.json" + file2 = tmpdir1 / "XRP_ETH-1m.json" + file2_new = tmpdir1 / "XRP_ETH-1m.json.gz" + + copyfile(file1_orig, file1) + copyfile(file2_orig, file2) + + default_conf['datadir'] = tmpdir1 default_conf['pairs'] = ['XRP_ETH'] default_conf['timeframes'] = ['1m', '5m'] @@ -317,10 +327,3 @@ def test_convert_ohlcv_format(mocker, default_conf, testdatadir): assert file2.exists() assert not file1_new.exists() assert not file2_new.exists() - - _clean_test_file(file1) - _clean_test_file(file2) - if file1_new.exists(): - file1_new.unlink() - if file2_new.exists(): - file2_new.unlink() diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index f9dac3397..f5c9a5a24 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -51,7 +51,7 @@ def test_text_table_bt_results(): assert text_table_bt_results(pair_results, stake_currency='BTC') == result_str -def test_generate_backtest_stats(default_conf, testdatadir): +def test_generate_backtest_stats(default_conf, testdatadir, tmpdir): default_conf.update({'strategy': 'DefaultStrategy'}) StrategyResolver.load_strategy(default_conf) @@ -148,8 +148,8 @@ def test_generate_backtest_stats(default_conf, testdatadir): assert strat_stats['pairlist'] == ['UNITTEST/BTC'] # Test storing stats - filename = Path(testdatadir / 'btresult.json') - filename_last = Path(testdatadir / LAST_BT_RESULT_FN) + filename = Path(tmpdir / 'btresult.json') + filename_last = Path(tmpdir / LAST_BT_RESULT_FN) _backup_file(filename_last, copy_file=True) assert not filename.is_file() @@ -159,7 +159,7 @@ def test_generate_backtest_stats(default_conf, testdatadir): last_fn = get_latest_backtest_filename(filename_last.parent) assert re.match(r"btresult-.*\.json", last_fn) - filename1 = (testdatadir / last_fn) + filename1 = Path(tmpdir / last_fn) assert filename1.is_file() content = filename1.read_text() assert 'max_drawdown' in content From b54da430b96a282458c7c836a7b5c35a345f9c51 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sun, 23 May 2021 11:22:59 +0300 Subject: [PATCH 051/127] Add ability to plot bars on indicator chart and pass custom arguments to plotly. --- docs/plotting.md | 14 ++++++++++++-- freqtrade/plot/plotting.py | 28 +++++++++++++++++++++------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index 5d454c414..05708ce66 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -170,9 +170,15 @@ Additional features when using plot_config include: * Specify additional subplots * Specify indicator pairs to fill area in between -The sample plot configuration below specifies fixed colors for the indicators. Otherwise consecutive plots may produce different colorschemes each time, making comparisons difficult. +The sample plot configuration below specifies fixed colors for the indicators. Otherwise, consecutive plots may produce different color schemes each time, making comparisons difficult. It also allows multiple subplots to display both MACD and RSI at the same time. +Plot type can be configured using `type` key. Possible types are: +* `scatter` corresponding to `plotly.graph_objects.Scatter` class (default). +* `bar` corresponding to `plotly.graph_objects.Bar` class. + +Extra parameters to `plotly.graph_objects.*` constructor can be specified in `plotly` dict. + Sample configuration with inline comments explaining the process: ``` python @@ -198,7 +204,8 @@ Sample configuration with inline comments explaining the process: # Create subplot MACD "MACD": { 'macd': {'color': 'blue', 'fill_to': 'macdhist'}, - 'macdsignal': {'color': 'orange'} + 'macdsignal': {'color': 'orange'}, + 'macdhist': {'type': 'bar', 'plotly': {'opacity': 0.9}} }, # Additional subplot RSI "RSI": { @@ -213,6 +220,9 @@ Sample configuration with inline comments explaining the process: The above configuration assumes that `ema10`, `ema50`, `senkou_a`, `senkou_b`, `macd`, `macdsignal`, `macdhist` and `rsi` are columns in the DataFrame created by the strategy. +!!! Warning + `plotly` arguments are only supported with plotly library and will not work with freq-ui. + ## Plot profit ![plot-profit](assets/plot-profit.png) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index bb4283406..194c20714 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -96,20 +96,34 @@ def add_indicators(fig, row, indicators: Dict[str, Dict], data: pd.DataFrame) -> Dict key must correspond to dataframe column. :param data: candlestick DataFrame """ + plot_kinds = { + 'scatter': go.Scatter, + 'bar': go.Bar, + } for indicator, conf in indicators.items(): logger.debug(f"indicator {indicator} with config {conf}") if indicator in data: kwargs = {'x': data['date'], 'y': data[indicator].values, - 'mode': 'lines', 'name': indicator } - if 'color' in conf: - kwargs.update({'line': {'color': conf['color']}}) - scatter = go.Scatter( - **kwargs - ) - fig.add_trace(scatter, row, 1) + + plot_type = conf.get('type', 'scatter') + color = conf.get('color') + if plot_type == 'bar': + kwargs.update({'marker_color': color or 'DarkSlateGrey', + 'marker_line_color': color or 'DarkSlateGrey'}) + else: + if color: + kwargs.update({'line': {'color': color}}) + kwargs['mode'] = 'lines' + if plot_type != 'scatter': + logger.warning(f'Indicator {indicator} has hnknown plot trace kind {plot_type}' + f', assuming "scatter".') + + kwargs.update(conf.get('plotly', {})) + trace = plot_kinds[plot_type](**kwargs) + fig.add_trace(trace, row, 1) else: logger.info( 'Indicator "%s" ignored. Reason: This indicator is not found ' From 806838c3af9981089af992ec36b90a9a377727ce Mon Sep 17 00:00:00 2001 From: Kamontat Chantrachirathumrong Date: Sun, 30 May 2021 21:07:44 +0700 Subject: [PATCH 052/127] Fix we use check sell_noti not noti --- freqtrade/rpc/telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index cca87ad91..d1e337401 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -240,8 +240,8 @@ class Telegram(RPCHandler): if msg_type == RPCMessageType.SELL: sell_noti = self._config['telegram'] \ .get('notification_settings', {}).get(str(msg_type), {}) - # For backward compatibility sell still be string - if isinstance(noti, str): + # For backward compatibility sell still can be string + if isinstance(sell_noti, str): noti = sell_noti else: noti = sell_noti.get(str(msg['sell_reason']), default_noti) From 901d984ee33f6abdff702082231189bf1cdae445 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 May 2021 15:50:37 +0200 Subject: [PATCH 053/127] Tests should write to tmpdir, not testdir --- tests/data/test_converter.py | 1 - tests/data/test_history.py | 131 +++++++++++++++-------------------- 2 files changed, 54 insertions(+), 78 deletions(-) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 31ce7255e..802fd4b12 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -262,7 +262,6 @@ def test_convert_trades_format(default_conf, testdatadir, tmpdir): ] for file in files: copyfile(testdatadir / file['old'].name, file['old']) - # _backup_file(file['old'], copy_file=True) assert not file['new'].exists() default_conf['datadir'] = tmpdir1 diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 353cfc6f7..d203d0792 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -86,14 +86,12 @@ def test_load_data_7min_timeframe(mocker, caplog, default_conf, testdatadir) -> def test_load_data_1min_timeframe(ohlcv_history, mocker, caplog, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history) file = testdatadir / 'UNITTEST_BTC-1m.json' - _backup_file(file, copy_file=True) load_data(datadir=testdatadir, timeframe='1m', pairs=['UNITTEST/BTC']) assert file.is_file() assert not log_has( 'Download history data for pair: "UNITTEST/BTC", interval: 1m ' 'and store in None.', caplog ) - _clean_test_file(file) def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: @@ -112,17 +110,17 @@ def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, - default_conf, testdatadir) -> None: + default_conf, tmpdir) -> None: """ Test load_pair_history() with 1 min timeframe """ + tmpdir1 = Path(tmpdir) mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) - file = testdatadir / 'MEME_BTC-1m.json' + file = tmpdir1 / 'MEME_BTC-1m.json' - _backup_file(file) # do not download a new pair if refresh_pairs isn't set - load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') + load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') assert not file.is_file() assert log_has( 'No history data for pair: "MEME/BTC", timeframe: 1m. ' @@ -130,15 +128,14 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, ) # download a new pair if refresh_pairs is set - refresh_data(datadir=testdatadir, timeframe='1m', pairs=['MEME/BTC'], + refresh_data(datadir=tmpdir1, timeframe='1m', pairs=['MEME/BTC'], exchange=exchange) - load_pair_history(datadir=testdatadir, timeframe='1m', pair='MEME/BTC') + load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') assert file.is_file() assert log_has_re( 'Download history data for pair: "MEME/BTC", timeframe: 1m ' 'and store in .*', caplog ) - _clean_test_file(file) def test_testdata_path(testdatadir) -> None: @@ -231,26 +228,22 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: assert start_ts is None -def test_download_pair_history(ohlcv_history_list, mocker, default_conf, testdatadir) -> None: +def test_download_pair_history(ohlcv_history_list, mocker, default_conf, tmpdir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) - file1_1 = testdatadir / 'MEME_BTC-1m.json' - file1_5 = testdatadir / 'MEME_BTC-5m.json' - file2_1 = testdatadir / 'CFI_BTC-1m.json' - file2_5 = testdatadir / 'CFI_BTC-5m.json' - - _backup_file(file1_1) - _backup_file(file1_5) - _backup_file(file2_1) - _backup_file(file2_5) + tmpdir1 = Path(tmpdir) + file1_1 = tmpdir1 / 'MEME_BTC-1m.json' + file1_5 = tmpdir1 / 'MEME_BTC-5m.json' + file2_1 = tmpdir1 / 'CFI_BTC-1m.json' + file2_5 = tmpdir1 / 'CFI_BTC-5m.json' assert not file1_1.is_file() assert not file2_1.is_file() - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', timeframe='1m') - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='CFI/BTC', timeframe='1m') assert not exchange._pairs_last_refresh_time @@ -264,20 +257,16 @@ def test_download_pair_history(ohlcv_history_list, mocker, default_conf, testdat assert not file1_5.is_file() assert not file2_5.is_file() - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', timeframe='5m') - assert _download_pair_history(datadir=testdatadir, exchange=exchange, + assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='CFI/BTC', timeframe='5m') assert not exchange._pairs_last_refresh_time assert file1_5.is_file() assert file2_5.is_file() - # clean files freshly downloaded - _clean_test_file(file1_5) - _clean_test_file(file2_5) - def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: tick = [ @@ -294,24 +283,15 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: assert json_dump_mock.call_count == 2 -def test_download_backtesting_data_exception(ohlcv_history, mocker, caplog, - default_conf, testdatadir) -> None: +def test_download_backtesting_data_exception(mocker, caplog, default_conf, tmpdir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', side_effect=Exception('File Error')) - + tmpdir1 = Path(tmpdir) exchange = get_patched_exchange(mocker, default_conf) - file1_1 = testdatadir / 'MEME_BTC-1m.json' - file1_5 = testdatadir / 'MEME_BTC-5m.json' - _backup_file(file1_1) - _backup_file(file1_5) - - assert not _download_pair_history(datadir=testdatadir, exchange=exchange, + assert not _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', timeframe='1m') - # clean files freshly downloaded - _clean_test_file(file1_1) - _clean_test_file(file1_5) assert log_has('Failed to download history data for pair: "MEME/BTC", timeframe: 1m.', caplog) @@ -528,15 +508,15 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes assert log_has("Skipping pair XRP/ETH...", caplog) -def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog) -> None: - +def test_download_trades_history(trades_history, mocker, default_conf, testdatadir, caplog, + tmpdir) -> None: + tmpdir1 = Path(tmpdir) ght_mock = MagicMock(side_effect=lambda pair, *args, **kwargs: (pair, trades_history)) mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', ght_mock) exchange = get_patched_exchange(mocker, default_conf) - file1 = testdatadir / 'ETH_BTC-trades.json.gz' - data_handler = get_datahandler(testdatadir, data_format='jsongz') - _backup_file(file1) + file1 = tmpdir1 / 'ETH_BTC-trades.json.gz' + data_handler = get_datahandler(tmpdir1, data_format='jsongz') assert not file1.is_file() @@ -557,8 +537,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert int(ght_mock.call_args_list[0][1]['since'] // 1000) == since_time2 - 5 assert ght_mock.call_args_list[0][1]['from_id'] is not None - # clean files freshly downloaded - _clean_test_file(file1) + file1.unlink() mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', MagicMock(side_effect=ValueError)) @@ -567,9 +546,8 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad pair='ETH/BTC') assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) - file2 = testdatadir / 'XRP_ETH-trades.json.gz' - - _backup_file(file2, True) + file2 = tmpdir1 / 'XRP_ETH-trades.json.gz' + copyfile(testdatadir / file2.name, file2) ght_mock.reset_mock() mocker.patch('freqtrade.exchange.Exchange.get_historic_trades', @@ -589,38 +567,37 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad _clean_test_file(file2) -def test_convert_trades_to_ohlcv(mocker, default_conf, testdatadir, caplog): - +def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog): + tmpdir1 = Path(tmpdir) pair = 'XRP/ETH' - file1 = testdatadir / 'XRP_ETH-1m.json' - file5 = testdatadir / 'XRP_ETH-5m.json' - # Compare downloaded dataset with converted dataset - dfbak_1m = load_pair_history(datadir=testdatadir, timeframe="1m", pair=pair) - dfbak_5m = load_pair_history(datadir=testdatadir, timeframe="5m", pair=pair) + file1 = tmpdir1 / 'XRP_ETH-1m.json' + file5 = tmpdir1 / 'XRP_ETH-5m.json' + filetrades = tmpdir1 / 'XRP_ETH-trades.json.gz' + copyfile(testdatadir / file1.name, file1) + copyfile(testdatadir / file5.name, file5) + copyfile(testdatadir / filetrades.name, filetrades) - _backup_file(file1, copy_file=True) - _backup_file(file5) + # Compare downloaded dataset with converted dataset + dfbak_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) + dfbak_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) tr = TimeRange.parse_timerange('20191011-20191012') convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'], - datadir=testdatadir, timerange=tr, erase=True) + datadir=tmpdir1, timerange=tr, erase=True) assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog) # Load new data - df_1m = load_pair_history(datadir=testdatadir, timeframe="1m", pair=pair) - df_5m = load_pair_history(datadir=testdatadir, timeframe="5m", pair=pair) + df_1m = load_pair_history(datadir=tmpdir1, timeframe="1m", pair=pair) + df_5m = load_pair_history(datadir=tmpdir1, timeframe="5m", pair=pair) assert df_1m.equals(dfbak_1m) assert df_5m.equals(dfbak_5m) - _clean_test_file(file1) - _clean_test_file(file5) - assert not log_has('Could not convert NoDatapair to OHLCV.', caplog) convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], - datadir=testdatadir, timerange=tr, erase=True) + datadir=tmpdir1, timerange=tr, erase=True) assert log_has('Could not convert NoDatapair to OHLCV.', caplog) @@ -752,15 +729,17 @@ def test_hdf5datahandler_trades_load(testdatadir): assert len([t for t in trades2 if t[0] > timerange.stopts * 1000]) == 0 -def test_hdf5datahandler_trades_store(testdatadir): +def test_hdf5datahandler_trades_store(testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) dh = HDF5DataHandler(testdatadir) trades = dh.trades_load('XRP/ETH') - dh.trades_store('XRP/NEW', trades) - file = testdatadir / 'XRP_NEW-trades.h5' + dh1 = HDF5DataHandler(tmpdir1) + dh1.trades_store('XRP/NEW', trades) + file = tmpdir1 / 'XRP_NEW-trades.h5' assert file.is_file() # Load trades back - trades_new = dh.trades_load('XRP/NEW') + trades_new = dh1.trades_load('XRP/NEW') assert len(trades_new) == len(trades) assert trades[0][0] == trades_new[0][0] @@ -778,8 +757,6 @@ def test_hdf5datahandler_trades_store(testdatadir): assert trades[-1][5] == trades_new[-1][5] assert trades[-1][6] == trades_new[-1][6] - _clean_test_file(file) - def test_hdf5datahandler_trades_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) @@ -793,16 +770,18 @@ def test_hdf5datahandler_trades_purge(mocker, testdatadir): assert unlinkmock.call_count == 1 -def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir): +def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir, tmpdir): + tmpdir1 = Path(tmpdir) dh = HDF5DataHandler(testdatadir) ohlcv = dh.ohlcv_load('UNITTEST/BTC', '5m') assert isinstance(ohlcv, DataFrame) assert len(ohlcv) > 0 - file = testdatadir / 'UNITTEST_NEW-5m.h5' + file = tmpdir1 / 'UNITTEST_NEW-5m.h5' assert not file.is_file() - dh.ohlcv_store('UNITTEST/NEW', '5m', ohlcv) + dh1 = HDF5DataHandler(tmpdir1) + dh1.ohlcv_store('UNITTEST/NEW', '5m', ohlcv) assert file.is_file() assert not ohlcv[ohlcv['date'] < '2018-01-15'].empty @@ -812,14 +791,12 @@ def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir): # Call private function to ensure timerange is filtered in hdf5 ohlcv = dh._ohlcv_load('UNITTEST/BTC', '5m', timerange) - ohlcv1 = dh._ohlcv_load('UNITTEST/NEW', '5m', timerange) + ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', '5m', timerange) assert len(ohlcv) == len(ohlcv1) assert ohlcv.equals(ohlcv1) assert ohlcv[ohlcv['date'] < '2018-01-15'].empty assert ohlcv[ohlcv['date'] > '2018-01-19'].empty - _clean_test_file(file) - # Try loading inexisting file ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', '5m') assert ohlcv.empty From e3d5c9cb10f8ba6f7eef554c7d13e6e0af895161 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 May 2021 16:39:33 +0100 Subject: [PATCH 054/127] Fix typo in exception message --- freqtrade/plot/plotting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 194c20714..b62ae6015 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -118,7 +118,7 @@ def add_indicators(fig, row, indicators: Dict[str, Dict], data: pd.DataFrame) -> kwargs.update({'line': {'color': color}}) kwargs['mode'] = 'lines' if plot_type != 'scatter': - logger.warning(f'Indicator {indicator} has hnknown plot trace kind {plot_type}' + logger.warning(f'Indicator {indicator} has unknown plot trace kind {plot_type}' f', assuming "scatter".') kwargs.update(conf.get('plotly', {})) From 06b59551b018fd4b7d066505e71e2510cd3e3b4e Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 May 2021 20:07:57 +0200 Subject: [PATCH 055/127] Improve test coverage --- tests/test_configuration.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index b2c883108..c15a0fe6a 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -88,6 +88,18 @@ def test_load_config_file_error_range(default_conf, mocker, caplog) -> None: '"stake_amount": .001, "fiat_display_currency": "USD", ' '"timeframe": "5m", "dry_run": true, "cance') + filedata = json.dumps(default_conf, indent=2).replace( + '"stake_amount": 0.001,', '"stake_amount": .001,') + mocker.patch.object(Path, "read_text", MagicMock(return_value=filedata)) + + x = log_config_error_range('somefile', 'Parse error at offset 4: Invalid value.') + assert isinstance(x, str) + assert (x == ' "max_open_trades": 1,\n "stake_currency": "BTC",\n' + ' "stake_amount": .001,') + + x = log_config_error_range('-', '') + assert x == '' + def test__args_to_config(caplog): From 2d7ccaeb3d019d810558c7a56dc97fcb5454786d Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 May 2021 20:14:41 +0200 Subject: [PATCH 056/127] Add test for load_config --- freqtrade/configuration/load_config.py | 2 +- tests/test_configuration.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/freqtrade/configuration/load_config.py b/freqtrade/configuration/load_config.py index 1320a375f..27190d259 100644 --- a/freqtrade/configuration/load_config.py +++ b/freqtrade/configuration/load_config.py @@ -43,7 +43,7 @@ def load_file(path: Path) -> Dict[str, Any]: with path.open('r') as file: config = rapidjson.load(file, parse_mode=CONFIG_PARSE_MODE) except FileNotFoundError: - raise OperationalException(f'File file "{path}" not found!') + raise OperationalException(f'File "{path}" not found!') return config diff --git a/tests/test_configuration.py b/tests/test_configuration.py index c15a0fe6a..b08d0775c 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -18,7 +18,7 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti process_deprecated_setting, process_removed_setting, process_temporary_deprecated_settings) -from freqtrade.configuration.load_config import load_config_file, log_config_error_range +from freqtrade.configuration.load_config import load_config_file, load_file, log_config_error_range from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL from freqtrade.exceptions import OperationalException from freqtrade.loggers import _set_loggers, setup_logging, setup_logging_pre @@ -101,6 +101,12 @@ def test_load_config_file_error_range(default_conf, mocker, caplog) -> None: assert x == '' +def test_load_file_error(tmpdir): + testpath = Path(tmpdir) / 'config.json' + with pytest.raises(OperationalException, match=r"File .* not found!"): + load_file(testpath) + + def test__args_to_config(caplog): arg_list = ['trade', '--strategy-path', 'TestTest'] From cd300c52ee7d5201b4b27b5e440934d0bacc568f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 05:18:19 +0000 Subject: [PATCH 057/127] Bump urllib3 from 1.26.4 to 1.26.5 Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.4 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.4...1.26.5) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 512b2a588..217d751c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ python-telegram-bot==13.5 arrow==1.1.0 cachetools==4.2.2 requests==2.25.1 -urllib3==1.26.4 +urllib3==1.26.5 wrapt==1.12.1 jsonschema==3.2.0 TA-Lib==0.4.20 From eb166147c3836b0ae4e3e677eb548eef57354326 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 05:18:39 +0000 Subject: [PATCH 058/127] Bump mkdocs-material from 7.1.5 to 7.1.6 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.1.5 to 7.1.6. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.1.5...7.1.6) Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 89011272d..3f8020e8c 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,3 @@ -mkdocs-material==7.1.5 +mkdocs-material==7.1.6 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From b4319b5ad849cc615e62b2d5406a4d55d0b88633 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 05:18:49 +0000 Subject: [PATCH 059/127] Bump sqlalchemy from 1.4.15 to 1.4.17 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.15 to 1.4.17. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 512b2a588..1b0b40698 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ ccxt==1.50.48 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.15 +SQLAlchemy==1.4.17 python-telegram-bot==13.5 arrow==1.1.0 cachetools==4.2.2 From 5d4e18233620812f4af1fe93e3665b96063b7564 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 05:18:54 +0000 Subject: [PATCH 060/127] Bump coveralls from 3.0.1 to 3.1.0 Bumps [coveralls](https://github.com/TheKevJames/coveralls-python) from 3.0.1 to 3.1.0. - [Release notes](https://github.com/TheKevJames/coveralls-python/releases) - [Changelog](https://github.com/TheKevJames/coveralls-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/TheKevJames/coveralls-python/compare/3.0.1...3.1.0) Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4fbf21260..7b106ae9b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,7 @@ -r requirements-plot.txt -r requirements-hyperopt.txt -coveralls==3.0.1 +coveralls==3.1.0 flake8==3.9.2 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.3.0 From f9541d301f767905f4a106e47f1dfdffb95c76b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 May 2021 08:19:14 +0000 Subject: [PATCH 061/127] Bump ccxt from 1.50.48 to 1.50.70 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.50.48 to 1.50.70. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.50.48...1.50.70) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 975adbdbf..8f4fed9ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.20.3 pandas==1.2.4 -ccxt==1.50.48 +ccxt==1.50.70 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From b5e3fe3b8e08a4ed7fedec2ef81453e57365cafc Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 31 May 2021 17:38:41 +0200 Subject: [PATCH 062/127] Document bittrex volumepairlist incompatibility closes #5051 --- docs/exchanges.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/exchanges.md b/docs/exchanges.md index 4a60c7683..e54f97714 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -53,6 +53,9 @@ Due to the heavy rate-limiting applied by Kraken, the following configuration se Bittrex does not support market orders. If you have a message at the bot startup about this, you should change order type values set in your configuration and/or in the strategy from `"market"` to `"limit"`. See some more details on this [here in the FAQ](faq.md#im-getting-the-exchange-bittrex-does-not-support-market-orders-message-and-cannot-run-my-strategy). +Bittrex also does not support `VolumePairlist` due to limited / split API constellation at the moment. +Please use `StaticPairlist`. Other pairlists (other than `VolumePairlist`) should not be affected. + ### Restricted markets Bittrex split its exchange into US and International versions. From f920c26802c8cccdc0ce4a29cbcae91711a26c55 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 31 May 2021 20:00:47 +0200 Subject: [PATCH 063/127] fix Hyperopt-list avg-time filters These should use a numeric field (which currently isn't available). closes #5061 --- freqtrade/commands/hyperopt_commands.py | 8 ++++++-- freqtrade/optimize/optimize_reports.py | 19 +++++++++++++------ tests/commands/test_commands.py | 9 +++++---- tests/conftest.py | 8 +++++++- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index e072e12cb..d8b00f369 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -197,8 +197,12 @@ def _hyperopt_filter_epochs_duration(epochs: List, filteroptions: dict) -> List: return x['results_metrics']['duration'] else: # New mode - avg = x['results_metrics']['holding_avg'] - return avg.total_seconds() // 60 + if 'holding_avg_s' in x['results_metrics']: + avg = x['results_metrics']['holding_avg_s'] + return avg // 60 + raise OperationalException( + "Holding-average not available. Please omit the filter on average time, " + "or rerun hyperopt with this version") if filteroptions['filter_min_avg_time'] is not None: epochs = _hyperopt_filter_epochs_trade(epochs, 0) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 5822fc627..84e052ac4 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -232,16 +232,23 @@ def generate_trading_stats(results: DataFrame) -> Dict[str, Any]: zero_duration_trades = len(results.loc[(results['trade_duration'] == 0) & (results['sell_reason'] == 'trailing_stop_loss')]) + holding_avg = (timedelta(minutes=round(results['trade_duration'].mean())) + if not results.empty else timedelta()) + winner_holding_avg = (timedelta(minutes=round(winning_trades['trade_duration'].mean())) + if not winning_trades.empty else timedelta()) + loser_holding_avg = (timedelta(minutes=round(losing_trades['trade_duration'].mean())) + if not losing_trades.empty else timedelta()) + return { 'wins': len(winning_trades), 'losses': len(losing_trades), 'draws': len(draw_trades), - 'holding_avg': (timedelta(minutes=round(results['trade_duration'].mean())) - if not results.empty else timedelta()), - 'winner_holding_avg': (timedelta(minutes=round(winning_trades['trade_duration'].mean())) - if not winning_trades.empty else timedelta()), - 'loser_holding_avg': (timedelta(minutes=round(losing_trades['trade_duration'].mean())) - if not losing_trades.empty else timedelta()), + 'holding_avg': holding_avg, + 'holding_avg_s': holding_avg.total_seconds(), + 'winner_holding_avg': winner_holding_avg, + 'winner_holding_avg_s': winner_holding_avg.total_seconds(), + 'loser_holding_avg': loser_holding_avg, + 'loser_holding_avg_s': loser_holding_avg.total_seconds(), 'zero_duration_trades': zero_duration_trades, } diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index f47fc46c1..71ae0ed78 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -921,10 +921,10 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, saved_hyperopt_results_legacy, tmpdir): csv_file = Path(tmpdir) / "test.csv" - for _ in (saved_hyperopt_results, saved_hyperopt_results_legacy): + for res in (saved_hyperopt_results, saved_hyperopt_results_legacy): mocker.patch( 'freqtrade.optimize.hyperopt_tools.HyperoptTools.load_previous_results', - MagicMock(return_value=saved_hyperopt_results_legacy) + MagicMock(return_value=res) ) args = [ @@ -1148,9 +1148,10 @@ def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, start_hyperopt_list(pargs) captured = capsys.readouterr() log_has("CSV file created: test_file.csv", caplog) - assert ('Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' - in csv_file.read_text()) assert csv_file.is_file() + line = csv_file.read_text() + assert ('Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,"3,930.0 m",0.43662' in line + or "Best,1,2,-1.25%,-1.2222,-0.00125625,,-2.51,2 days 17:30:00,0.43662" in line) csv_file.unlink() diff --git a/tests/conftest.py b/tests/conftest.py index ef2bd0613..43a98647f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1913,7 +1913,7 @@ def saved_hyperopt_results_legacy(): @pytest.fixture def saved_hyperopt_results(): - return [ + hyperopt_res = [ { 'loss': 0.4366182531161519, 'params_dict': { @@ -2042,3 +2042,9 @@ def saved_hyperopt_results(): 'is_best': False } ] + + for res in hyperopt_res: + res['results_metrics']['holding_avg_s'] = res['results_metrics']['holding_avg' + ].total_seconds() + + return hyperopt_res From 53b1f38952a698c2fd5e3661c2125ae0014c1d91 Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Tue, 1 Jun 2021 20:08:22 +0300 Subject: [PATCH 064/127] telegram: move the most important information to the top of sell message --- freqtrade/rpc/telegram.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index d1e337401..ccf19add1 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -211,23 +211,26 @@ class Telegram(RPCHandler): msg['emoji'] = self._get_sell_emoji(msg) - message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n" - "*Amount:* `{amount:.8f}`\n" - "*Open Rate:* `{open_rate:.8f}`\n" - "*Current Rate:* `{current_rate:.8f}`\n" - "*Close Rate:* `{limit:.8f}`\n" - "*Sell Reason:* `{sell_reason}`\n" - "*Duration:* `{duration} ({duration_min:.1f} min)`\n" - "*Profit:* `{profit_percent:.2f}%`").format(**msg) - # Check if all sell properties are available. # This might not be the case if the message origin is triggered by /forcesell if (all(prop in msg for prop in ['gain', 'fiat_currency', 'stake_currency']) and self._rpc._fiat_converter): msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount( msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) - message += (' `({gain}: {profit_amount:.8f} {stake_currency}' - ' / {profit_fiat:.3f} {fiat_currency})`').format(**msg) + msg['profit_extra'] = (' ({gain}: {profit_amount:.8f} {stake_currency}' + ' / {profit_fiat:.3f} {fiat_currency})').format(**msg) + else: + msg['profit_extra'] = '' + + message = ("{emoji} *{exchange}:* Selling {pair} (#{trade_id})\n" + "*Profit:* `{profit_percent:.2f}%{profit_extra}`\n" + "*Sell Reason:* `{sell_reason}`\n" + "*Duration:* `{duration} ({duration_min:.1f} min)`\n" + "*Amount:* `{amount:.8f}`\n" + "*Open Rate:* `{open_rate:.8f}`\n" + "*Current Rate:* `{current_rate:.8f}`\n" + "*Close Rate:* `{limit:.8f}`").format(**msg) + return message def send_msg(self, msg: Dict[str, Any]) -> None: From 79552a93fe4924b59cac6d68f2a207f42b2f28d0 Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Tue, 1 Jun 2021 20:17:11 +0300 Subject: [PATCH 065/127] telegram: move the most important information to the top of sell message fixed tests --- tests/rpc/test_rpc_telegram.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index e640f2dff..4c60bdad3 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -1350,13 +1350,14 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n' + '*Profit:* `-57.41% (loss: -0.05746268 ETH / -24.812 USD)`\n' + '*Sell Reason:* `stop_loss`\n' + '*Duration:* `1:00:00 (60.0 min)`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' - '*Close Rate:* `0.00003201`\n' - '*Sell Reason:* `stop_loss`\n' - '*Duration:* `1:00:00 (60.0 min)`\n' - '*Profit:* `-57.41%` `(loss: -0.05746268 ETH / -24.812 USD)`') + '*Close Rate:* `0.00003201`' + ) msg_mock.reset_mock() telegram.send_msg({ @@ -1379,13 +1380,14 @@ def test_send_msg_sell_notification(default_conf, mocker) -> None: }) assert msg_mock.call_args[0][0] \ == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n' + '*Profit:* `-57.41%`\n' + '*Sell Reason:* `stop_loss`\n' + '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' - '*Close Rate:* `0.00003201`\n' - '*Sell Reason:* `stop_loss`\n' - '*Duration:* `1 day, 2:30:00 (1590.0 min)`\n' - '*Profit:* `-57.41%`') + '*Close Rate:* `0.00003201`' + ) # Reset singleton function to avoid random breaks telegram._rpc._fiat_converter.convert_amount = old_convamount @@ -1537,13 +1539,14 @@ def test_send_msg_sell_notification_no_fiat(default_conf, mocker) -> None: 'close_date': arrow.utcnow(), }) assert msg_mock.call_args[0][0] == ('\N{WARNING SIGN} *Binance:* Selling KEY/ETH (#1)\n' + '*Profit:* `-57.41%`\n' + '*Sell Reason:* `stop_loss`\n' + '*Duration:* `2:35:03 (155.1 min)`\n' '*Amount:* `1333.33333333`\n' '*Open Rate:* `0.00007500`\n' '*Current Rate:* `0.00003201`\n' - '*Close Rate:* `0.00003201`\n' - '*Sell Reason:* `stop_loss`\n' - '*Duration:* `2:35:03 (155.1 min)`\n' - '*Profit:* `-57.41%`') + '*Close Rate:* `0.00003201`' + ) @pytest.mark.parametrize('msg,expected', [ From 1594402312c871b3a3df64eab142d699922e0147 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 1 Jun 2021 19:39:41 +0200 Subject: [PATCH 066/127] Add note about signal expiry --- docs/configuration.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index eff8fe322..7fa751bef 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -304,6 +304,9 @@ For example, if your strategy is using a 1h timeframe, and you only want to buy }, ``` +!!! Note + This setting resets with each new candle, so it will not prevent sticking-signals from executing on the 2nd or 3rd candle they're active. Best use a "trigger" selector for buy signals, which are only active for one candle. + ### Understand order_types The `order_types` configuration parameter maps actions (`buy`, `sell`, `stoploss`, `emergencysell`, `forcesell`, `forcebuy`) to order-types (`market`, `limit`, ...) as well as configures stoploss to be on the exchange and defines stoploss on exchange update interval in seconds. From 9edcb393b6704bebbffd0552e029f8126b2c7269 Mon Sep 17 00:00:00 2001 From: Eugene Schava Date: Tue, 1 Jun 2021 22:24:21 +0300 Subject: [PATCH 067/127] telegram: move the most important information to the top of sell message fixed flake error --- freqtrade/rpc/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ccf19add1..320a3a591 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -218,7 +218,7 @@ class Telegram(RPCHandler): msg['profit_fiat'] = self._rpc._fiat_converter.convert_amount( msg['profit_amount'], msg['stake_currency'], msg['fiat_currency']) msg['profit_extra'] = (' ({gain}: {profit_amount:.8f} {stake_currency}' - ' / {profit_fiat:.3f} {fiat_currency})').format(**msg) + ' / {profit_fiat:.3f} {fiat_currency})').format(**msg) else: msg['profit_extra'] = '' From 10cd89a99d4e75e7af03fda8b6fbb77cfe573c2a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 10:39:49 +0200 Subject: [PATCH 068/127] Allow the API to respond faster in case of long pairlists --- freqtrade/freqtradebot.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 2e3240cfe..d7369ad47 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -48,6 +48,7 @@ class FreqtradeBot(LoggingMixin): :param config: configuration dict, you can use Configuration.get_config() to get the config dict. """ + self.active_pair_whitelist: List[str] = [] logger.info('Starting freqtrade %s', __version__) @@ -76,12 +77,19 @@ class FreqtradeBot(LoggingMixin): PairLocks.timeframe = self.config['timeframe'] + self.protections = ProtectionManager(self.config) + + # RPC runs in separate threads, can start handling external commands just after + # initialization, even before Freqtradebot has a chance to start its throttling, + # so anything in the Freqtradebot instance should be ready (initialized), including + # the initial state of the bot. + # Keep this at the end of this initialization method. + self.rpc: RPCManager = RPCManager(self) + self.pairlists = PairListManager(self.exchange, self.config) self.dataprovider = DataProvider(self.config, self.exchange, self.pairlists) - self.protections = ProtectionManager(self.config) - # Attach Dataprovider to Strategy baseclass IStrategy.dp = self.dataprovider # Attach Wallets to Strategy baseclass @@ -97,12 +105,6 @@ class FreqtradeBot(LoggingMixin): initial_state = self.config.get('initial_state') self.state = State[initial_state.upper()] if initial_state else State.STOPPED - # RPC runs in separate threads, can start handling external commands just after - # initialization, even before Freqtradebot has a chance to start its throttling, - # so anything in the Freqtradebot instance should be ready (initialized), including - # the initial state of the bot. - # Keep this at the end of this initialization method. - self.rpc: RPCManager = RPCManager(self) # Protect sell-logic from forcesell and viceversa self._sell_lock = Lock() LoggingMixin.__init__(self, logger, timeframe_to_seconds(self.strategy.timeframe)) From 67beda6c929df56a1c9861202fdb839e5fb311e4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:06:32 +0200 Subject: [PATCH 069/127] Add fetch_dry_run_order method --- freqtrade/exchange/exchange.py | 28 ++++++++++++++++++---------- freqtrade/exchange/ftx.py | 9 ++------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 93d8f7584..02445de92 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -591,6 +591,19 @@ class Exchange: closed_order["info"].update({"stopPrice": closed_order["price"]}) self._dry_run_open_orders[closed_order["id"]] = closed_order + def fetch_dry_run_order(self, order_id) -> Dict[str, Any]: + """ + Return dry-run order + Only call if running in dry-run mode. + """ + try: + order = self._dry_run_open_orders[order_id] + return order + except KeyError as e: + # Gracefully handle errors with dry-run orders. + raise InvalidOrderException( + f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + def create_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict: try: @@ -1066,11 +1079,12 @@ class Exchange: @retrier def cancel_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - order = self._dry_run_open_orders.get(order_id) - if order: + try: + order = self.fetch_dry_run_order(order_id) + order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) return order - else: + except InvalidOrderException: return {} try: @@ -1144,13 +1158,7 @@ class Exchange: @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - try: - order = self._dry_run_open_orders[order_id] - return order - except KeyError as e: - # Gracefully handle errors with dry-run orders. - raise InvalidOrderException( - f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + return self.fetch_dry_run_order(order_id) try: return self._api.fetch_order(order_id, pair) except ccxt.OrderNotFound as e: diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 9009e9492..105389828 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -93,13 +93,8 @@ class Ftx(Exchange): @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) def fetch_stoploss_order(self, order_id: str, pair: str) -> Dict: if self._config['dry_run']: - try: - order = self._dry_run_open_orders[order_id] - return order - except KeyError as e: - # Gracefully handle errors with dry-run orders. - raise InvalidOrderException( - f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + return self.fetch_dry_run_order(order_id) + try: orders = self._api.fetch_orders(pair, None, params={'type': 'stop'}) From 4c277b3039bda30369ada1157368dd93d19e4dd8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:17:50 +0200 Subject: [PATCH 070/127] Reorder exchange methods --- freqtrade/exchange/exchange.py | 332 +++++++++++++++++---------------- 1 file changed, 169 insertions(+), 163 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 02445de92..c3c4e8e5a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -550,6 +550,8 @@ class Exchange: # See also #2575 at github. return max(min_stake_amounts) * amount_reserve_percent + # Dry-run methods + def create_dry_run_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict[str, Any]: order_id = f'dry_run_{side}_{datetime.now().timestamp()}' @@ -604,6 +606,8 @@ class Exchange: raise InvalidOrderException( f'Tried to get an invalid dry-run-order (id: {order_id}). Message: {e}') from e + # Order handling + def create_order(self, pair: str, ordertype: str, side: str, amount: float, rate: float, params: Dict = {}) -> Dict: try: @@ -680,6 +684,128 @@ class Exchange: raise OperationalException(f"stoploss is not implemented for {self.name}.") + @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) + def fetch_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + return self.fetch_dry_run_order(order_id) + try: + return self._api.fetch_order(order_id, pair) + except ccxt.OrderNotFound as e: + raise RetryableOrderError( + f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + # Assign method to fetch_stoploss_order to allow easy overriding in other classes + fetch_stoploss_order = fetch_order + + def fetch_order_or_stoploss_order(self, order_id: str, pair: str, + stoploss_order: bool = False) -> Dict: + """ + Simple wrapper calling either fetch_order or fetch_stoploss_order depending on + the stoploss_order parameter + :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. + """ + if stoploss_order: + return self.fetch_stoploss_order(order_id, pair) + return self.fetch_order(order_id, pair) + + def check_order_canceled_empty(self, order: Dict) -> bool: + """ + Verify if an order has been cancelled without being partially filled + :param order: Order dict as returned from fetch_order() + :return: True if order has been cancelled without being filled, False otherwise. + """ + return (order.get('status') in ('closed', 'canceled', 'cancelled') + and order.get('filled') == 0.0) + + @retrier + def cancel_order(self, order_id: str, pair: str) -> Dict: + if self._config['dry_run']: + try: + order = self.fetch_dry_run_order(order_id) + + order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) + return order + except InvalidOrderException: + return {} + + try: + return self._api.cancel_order(order_id, pair) + except ccxt.InvalidOrder as e: + raise InvalidOrderException( + f'Could not cancel order. Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + # Assign method to cancel_stoploss_order to allow easy overriding in other classes + cancel_stoploss_order = cancel_order + + def is_cancel_order_result_suitable(self, corder) -> bool: + if not isinstance(corder, dict): + return False + + required = ('fee', 'status', 'amount') + return all(k in corder for k in required) + + def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: + """ + Cancel order returning a result. + Creates a fake result if cancel order returns a non-usable result + and fetch_order does not work (certain exchanges don't return cancelled orders) + :param order_id: Orderid to cancel + :param pair: Pair corresponding to order_id + :param amount: Amount to use for fake response + :return: Result from either cancel_order if usable, or fetch_order + """ + try: + corder = self.cancel_order(order_id, pair) + if self.is_cancel_order_result_suitable(corder): + return corder + except InvalidOrderException: + logger.warning(f"Could not cancel order {order_id} for {pair}.") + try: + order = self.fetch_order(order_id, pair) + except InvalidOrderException: + logger.warning(f"Could not fetch cancelled order {order_id}.") + order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + + return order + + def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: + """ + Cancel stoploss order returning a result. + Creates a fake result if cancel order returns a non-usable result + and fetch_order does not work (certain exchanges don't return cancelled orders) + :param order_id: stoploss-order-id to cancel + :param pair: Pair corresponding to order_id + :param amount: Amount to use for fake response + :return: Result from either cancel_order if usable, or fetch_order + """ + corder = self.cancel_stoploss_order(order_id, pair) + if self.is_cancel_order_result_suitable(corder): + return corder + try: + order = self.fetch_stoploss_order(order_id, pair) + except InvalidOrderException: + logger.warning(f"Could not fetch cancelled stoploss order {order_id}.") + order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} + + return order + @retrier def get_balances(self) -> dict: @@ -726,6 +852,8 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + # Pricing info + @retrier def fetch_ticker(self, pair: str) -> dict: try: @@ -742,6 +870,47 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + @staticmethod + def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]], + range_required: bool = True): + """ + Get next greater value in the list. + Used by fetch_l2_order_book if the api only supports a limited range + """ + if not limit_range: + return limit + + result = min([x for x in limit_range if limit <= x] + [max(limit_range)]) + if not range_required and limit > result: + # Range is not required - we can use None as parameter. + return None + return result + + @retrier + def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: + """ + Get L2 order book from exchange. + Can be limited to a certain amount (if supported). + Returns a dict in the format + {'asks': [price, volume], 'bids': [price, volume]} + """ + limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'], + self._ft_has['l2_limit_range_required']) + try: + + return self._api.fetch_l2_order_book(pair, limit1) + except ccxt.NotSupported as e: + raise OperationalException( + f'Exchange {self._api.name} does not support fetching order book.' + f'Message: {e}') from e + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: """ @@ -1067,169 +1236,6 @@ class Exchange: self._async_get_trade_history(pair=pair, since=since, until=until, from_id=from_id)) - def check_order_canceled_empty(self, order: Dict) -> bool: - """ - Verify if an order has been cancelled without being partially filled - :param order: Order dict as returned from fetch_order() - :return: True if order has been cancelled without being filled, False otherwise. - """ - return (order.get('status') in ('closed', 'canceled', 'cancelled') - and order.get('filled') == 0.0) - - @retrier - def cancel_order(self, order_id: str, pair: str) -> Dict: - if self._config['dry_run']: - try: - order = self.fetch_dry_run_order(order_id) - - order.update({'status': 'canceled', 'filled': 0.0, 'remaining': order['amount']}) - return order - except InvalidOrderException: - return {} - - try: - return self._api.cancel_order(order_id, pair) - except ccxt.InvalidOrder as e: - raise InvalidOrderException( - f'Could not cancel order. Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not cancel order due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - # Assign method to cancel_stoploss_order to allow easy overriding in other classes - cancel_stoploss_order = cancel_order - - def is_cancel_order_result_suitable(self, corder) -> bool: - if not isinstance(corder, dict): - return False - - required = ('fee', 'status', 'amount') - return all(k in corder for k in required) - - def cancel_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: - """ - Cancel order returning a result. - Creates a fake result if cancel order returns a non-usable result - and fetch_order does not work (certain exchanges don't return cancelled orders) - :param order_id: Orderid to cancel - :param pair: Pair corresponding to order_id - :param amount: Amount to use for fake response - :return: Result from either cancel_order if usable, or fetch_order - """ - try: - corder = self.cancel_order(order_id, pair) - if self.is_cancel_order_result_suitable(corder): - return corder - except InvalidOrderException: - logger.warning(f"Could not cancel order {order_id} for {pair}.") - try: - order = self.fetch_order(order_id, pair) - except InvalidOrderException: - logger.warning(f"Could not fetch cancelled order {order_id}.") - order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} - - return order - - def cancel_stoploss_order_with_result(self, order_id: str, pair: str, amount: float) -> Dict: - """ - Cancel stoploss order returning a result. - Creates a fake result if cancel order returns a non-usable result - and fetch_order does not work (certain exchanges don't return cancelled orders) - :param order_id: stoploss-order-id to cancel - :param pair: Pair corresponding to order_id - :param amount: Amount to use for fake response - :return: Result from either cancel_order if usable, or fetch_order - """ - corder = self.cancel_stoploss_order(order_id, pair) - if self.is_cancel_order_result_suitable(corder): - return corder - try: - order = self.fetch_stoploss_order(order_id, pair) - except InvalidOrderException: - logger.warning(f"Could not fetch cancelled stoploss order {order_id}.") - order = {'fee': {}, 'status': 'canceled', 'amount': amount, 'info': {}} - - return order - - @retrier(retries=API_FETCH_ORDER_RETRY_COUNT) - def fetch_order(self, order_id: str, pair: str) -> Dict: - if self._config['dry_run']: - return self.fetch_dry_run_order(order_id) - try: - return self._api.fetch_order(order_id, pair) - except ccxt.OrderNotFound as e: - raise RetryableOrderError( - f'Order not found (pair: {pair} id: {order_id}). Message: {e}') from e - except ccxt.InvalidOrder as e: - raise InvalidOrderException( - f'Tried to get an invalid order (pair: {pair} id: {order_id}). Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get order due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - # Assign method to fetch_stoploss_order to allow easy overriding in other classes - fetch_stoploss_order = fetch_order - - def fetch_order_or_stoploss_order(self, order_id: str, pair: str, - stoploss_order: bool = False) -> Dict: - """ - Simple wrapper calling either fetch_order or fetch_stoploss_order depending on - the stoploss_order parameter - :param stoploss_order: If true, uses fetch_stoploss_order, otherwise fetch_order. - """ - if stoploss_order: - return self.fetch_stoploss_order(order_id, pair) - return self.fetch_order(order_id, pair) - - @staticmethod - def get_next_limit_in_list(limit: int, limit_range: Optional[List[int]], - range_required: bool = True): - """ - Get next greater value in the list. - Used by fetch_l2_order_book if the api only supports a limited range - """ - if not limit_range: - return limit - - result = min([x for x in limit_range if limit <= x] + [max(limit_range)]) - if not range_required and limit > result: - # Range is not required - we can use None as parameter. - return None - return result - - @retrier - def fetch_l2_order_book(self, pair: str, limit: int = 100) -> dict: - """ - Get L2 order book from exchange. - Can be limited to a certain amount (if supported). - Returns a dict in the format - {'asks': [price, volume], 'bids': [price, volume]} - """ - limit1 = self.get_next_limit_in_list(limit, self._ft_has['l2_limit_range'], - self._ft_has['l2_limit_range_required']) - try: - - return self._api.fetch_l2_order_book(pair, limit1) - except ccxt.NotSupported as e: - raise OperationalException( - f'Exchange {self._api.name} does not support fetching order book.' - f'Message: {e}') from e - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get order book due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - @retrier def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: """ From 4e1425023e0aa4c0530b5c2ffb8a2e2c20dfd772 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:20:26 +0200 Subject: [PATCH 071/127] Further reorder exchange methods --- freqtrade/exchange/exchange.py | 242 +++++++++++++++++---------------- 1 file changed, 124 insertions(+), 118 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index c3c4e8e5a..7514572fc 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -911,6 +911,128 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + # Fee handling + + @retrier + def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: + """ + Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. + The "since" argument passed in is coming from the database and is in UTC, + as timezone-native datetime object. + From the python documentation: + > Naive datetime instances are assumed to represent local time + Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the + transformation from local timezone to UTC. + This works for timezones UTC+ since then the result will contain trades from a few hours + instead of from the last 5 seconds, however fails for UTC- timezones, + since we're then asking for trades with a "since" argument in the future. + + :param order_id order_id: Order-id as given when creating the order + :param pair: Pair the order is for + :param since: datetime object of the order creation time. Assumes object is in UTC. + """ + if self._config['dry_run']: + return [] + if not self.exchange_has('fetchMyTrades'): + return [] + try: + # Allow 5s offset to catch slight time offsets (discovered in #1185) + # since needs to be int in milliseconds + my_trades = self._api.fetch_my_trades( + pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) + matched_trades = [trade for trade in my_trades if trade['order'] == order_id] + + return matched_trades + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + def get_order_id_conditional(self, order: Dict[str, Any]) -> str: + return order['id'] + + @retrier + def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, + price: float = 1, taker_or_maker: str = 'maker') -> float: + try: + if self._config['dry_run'] and self._config.get('fee', None) is not None: + return self._config['fee'] + # validate that markets are loaded before trying to get fee + if self._api.markets is None or len(self._api.markets) == 0: + self._api.load_markets() + + return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, + price=price, takerOrMaker=taker_or_maker)['rate'] + except ccxt.DDoSProtection as e: + raise DDosProtection(e) from e + except (ccxt.NetworkError, ccxt.ExchangeError) as e: + raise TemporaryError( + f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e + except ccxt.BaseError as e: + raise OperationalException(e) from e + + @staticmethod + def order_has_fee(order: Dict) -> bool: + """ + Verifies if the passed in order dict has the needed keys to extract fees, + and that these keys (currency, cost) are not empty. + :param order: Order or trade (one trade) dict + :return: True if the fee substructure contains currency and cost, false otherwise + """ + if not isinstance(order, dict): + return False + return ('fee' in order and order['fee'] is not None + and (order['fee'].keys() >= {'currency', 'cost'}) + and order['fee']['currency'] is not None + and order['fee']['cost'] is not None + ) + + def calculate_fee_rate(self, order: Dict) -> Optional[float]: + """ + Calculate fee rate if it's not given by the exchange. + :param order: Order or trade (one trade) dict + """ + if order['fee'].get('rate') is not None: + return order['fee'].get('rate') + fee_curr = order['fee']['currency'] + # Calculate fee based on order details + if fee_curr in self.get_pair_base_currency(order['symbol']): + # Base currency - divide by amount + return round( + order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8) + elif fee_curr in self.get_pair_quote_currency(order['symbol']): + # Quote currency - divide by cost + return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None + else: + # If Fee currency is a different currency + if not order['cost']: + # If cost is None or 0.0 -> falsy, return None + return None + try: + comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) + tick = self.fetch_ticker(comb) + + fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') + return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) + except ExchangeError: + return None + + def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: + """ + Extract tuple of cost, currency, rate. + Requires order_has_fee to run first! + :param order: Order or trade (one trade) dict + :return: Tuple with cost, currency, rate of the given fee dict + """ + return (order['fee']['cost'], + order['fee']['currency'], + self.calculate_fee_rate(order)) + + # Historic data + def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int) -> List: """ @@ -1078,6 +1200,8 @@ class Exchange: raise OperationalException(f'Could not fetch historical candle (OHLCV) data ' f'for pair {pair}. Message: {e}') from e + # Fetch historic trades + @retrier_async async def _async_fetch_trades(self, pair: str, since: Optional[int] = None, @@ -1236,124 +1360,6 @@ class Exchange: self._async_get_trade_history(pair=pair, since=since, until=until, from_id=from_id)) - @retrier - def get_trades_for_order(self, order_id: str, pair: str, since: datetime) -> List: - """ - Fetch Orders using the "fetch_my_trades" endpoint and filter them by order-id. - The "since" argument passed in is coming from the database and is in UTC, - as timezone-native datetime object. - From the python documentation: - > Naive datetime instances are assumed to represent local time - Therefore, calling "since.timestamp()" will get the UTC timestamp, after applying the - transformation from local timezone to UTC. - This works for timezones UTC+ since then the result will contain trades from a few hours - instead of from the last 5 seconds, however fails for UTC- timezones, - since we're then asking for trades with a "since" argument in the future. - - :param order_id order_id: Order-id as given when creating the order - :param pair: Pair the order is for - :param since: datetime object of the order creation time. Assumes object is in UTC. - """ - if self._config['dry_run']: - return [] - if not self.exchange_has('fetchMyTrades'): - return [] - try: - # Allow 5s offset to catch slight time offsets (discovered in #1185) - # since needs to be int in milliseconds - my_trades = self._api.fetch_my_trades( - pair, int((since.replace(tzinfo=timezone.utc).timestamp() - 5) * 1000)) - matched_trades = [trade for trade in my_trades if trade['order'] == order_id] - - return matched_trades - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get trades due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - def get_order_id_conditional(self, order: Dict[str, Any]) -> str: - return order['id'] - - @retrier - def get_fee(self, symbol: str, type: str = '', side: str = '', amount: float = 1, - price: float = 1, taker_or_maker: str = 'maker') -> float: - try: - if self._config['dry_run'] and self._config.get('fee', None) is not None: - return self._config['fee'] - # validate that markets are loaded before trying to get fee - if self._api.markets is None or len(self._api.markets) == 0: - self._api.load_markets() - - return self._api.calculate_fee(symbol=symbol, type=type, side=side, amount=amount, - price=price, takerOrMaker=taker_or_maker)['rate'] - except ccxt.DDoSProtection as e: - raise DDosProtection(e) from e - except (ccxt.NetworkError, ccxt.ExchangeError) as e: - raise TemporaryError( - f'Could not get fee info due to {e.__class__.__name__}. Message: {e}') from e - except ccxt.BaseError as e: - raise OperationalException(e) from e - - @staticmethod - def order_has_fee(order: Dict) -> bool: - """ - Verifies if the passed in order dict has the needed keys to extract fees, - and that these keys (currency, cost) are not empty. - :param order: Order or trade (one trade) dict - :return: True if the fee substructure contains currency and cost, false otherwise - """ - if not isinstance(order, dict): - return False - return ('fee' in order and order['fee'] is not None - and (order['fee'].keys() >= {'currency', 'cost'}) - and order['fee']['currency'] is not None - and order['fee']['cost'] is not None - ) - - def calculate_fee_rate(self, order: Dict) -> Optional[float]: - """ - Calculate fee rate if it's not given by the exchange. - :param order: Order or trade (one trade) dict - """ - if order['fee'].get('rate') is not None: - return order['fee'].get('rate') - fee_curr = order['fee']['currency'] - # Calculate fee based on order details - if fee_curr in self.get_pair_base_currency(order['symbol']): - # Base currency - divide by amount - return round( - order['fee']['cost'] / safe_value_fallback2(order, order, 'filled', 'amount'), 8) - elif fee_curr in self.get_pair_quote_currency(order['symbol']): - # Quote currency - divide by cost - return round(order['fee']['cost'] / order['cost'], 8) if order['cost'] else None - else: - # If Fee currency is a different currency - if not order['cost']: - # If cost is None or 0.0 -> falsy, return None - return None - try: - comb = self.get_valid_pair_combination(fee_curr, self._config['stake_currency']) - tick = self.fetch_ticker(comb) - - fee_to_quote_rate = safe_value_fallback2(tick, tick, 'last', 'ask') - return round((order['fee']['cost'] * fee_to_quote_rate) / order['cost'], 8) - except ExchangeError: - return None - - def extract_cost_curr_rate(self, order: Dict) -> Tuple[float, str, Optional[float]]: - """ - Extract tuple of cost, currency, rate. - Requires order_has_fee to run first! - :param order: Order or trade (one trade) dict - :return: Tuple with cost, currency, rate of the given fee dict - """ - return (order['fee']['cost'], - order['fee']['currency'], - self.calculate_fee_rate(order)) - def is_exchange_known_ccxt(exchange_name: str, ccxt_module: CcxtModuleType = None) -> bool: return exchange_name in ccxt_exchanges(ccxt_module) From 12916243ecf241bb3332fd36bff844cb49011717 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:30:19 +0200 Subject: [PATCH 072/127] Move get_buy_rate to exchange class --- freqtrade/exchange/exchange.py | 51 +++++++++++++++++++++++++++++++-- freqtrade/freqtradebot.py | 49 ++----------------------------- tests/exchange/test_exchange.py | 44 ++++++++++++++++++++++++++++ tests/test_freqtradebot.py | 43 --------------------------- 4 files changed, 95 insertions(+), 92 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7514572fc..42b518566 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -22,8 +22,8 @@ from pandas import DataFrame from freqtrade.constants import DEFAULT_AMOUNT_RESERVE_PERCENT, ListPairsWithTimeframes from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, - InvalidOrderException, OperationalException, RetryableOrderError, - TemporaryError) + InvalidOrderException, OperationalException, PricingError, + RetryableOrderError, TemporaryError) from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, BAD_EXCHANGES, EXCHANGE_HAS_OPTIONAL, EXCHANGE_HAS_REQUIRED, retrier, retrier_async) @@ -88,6 +88,7 @@ class Exchange: # Cache for 10 minutes ... self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10) + self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) # Holds candles self._klines: Dict[Tuple[str, str], DataFrame] = {} @@ -911,6 +912,52 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def get_buy_rate(self, pair: str, refresh: bool) -> float: + """ + Calculates bid target between current ask price and last price + :param pair: Pair to get rate for + :param refresh: allow cached data + :return: float: Price + :raises PricingError if orderbook price could not be determined. + """ + if not refresh: + rate = self._buy_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.debug(f"Using cached buy rate for {pair}.") + return rate + + bid_strategy = self._config.get('bid_strategy', {}) + if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): + + order_book_top = bid_strategy.get('order_book_top', 1) + order_book = self.fetch_l2_order_book(pair, order_book_top) + logger.debug('order_book %s', order_book) + # top 1 = index 0 + try: + rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] + except (IndexError, KeyError) as e: + logger.warning( + "Buy Price from orderbook could not be determined." + f"Orderbook: {order_book}" + ) + raise PricingError from e + logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side " + f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}") + used_rate = rate_from_l2 + else: + logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") + ticker = self.fetch_ticker(pair) + ticker_rate = ticker[bid_strategy['price_side']] + if ticker['last'] and ticker_rate > ticker['last']: + balance = bid_strategy['ask_last_balance'] + ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) + used_rate = ticker_rate + + self._buy_rate_cache[pair] = used_rate + + return used_rate + # Fee handling @retrier diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d7369ad47..d15748864 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -62,7 +62,6 @@ class FreqtradeBot(LoggingMixin): # Caching only applies to RPC methods, so prices for open trades are still # refreshed once every iteration. self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) - self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) @@ -396,50 +395,6 @@ class FreqtradeBot(LoggingMixin): return trades_created - def get_buy_rate(self, pair: str, refresh: bool) -> float: - """ - Calculates bid target between current ask price and last price - :param pair: Pair to get rate for - :param refresh: allow cached data - :return: float: Price - """ - if not refresh: - rate = self._buy_rate_cache.get(pair) - # Check if cache has been invalidated - if rate: - logger.debug(f"Using cached buy rate for {pair}.") - return rate - - bid_strategy = self.config.get('bid_strategy', {}) - if 'use_order_book' in bid_strategy and bid_strategy.get('use_order_book', False): - - order_book_top = bid_strategy.get('order_book_top', 1) - order_book = self.exchange.fetch_l2_order_book(pair, order_book_top) - logger.debug('order_book %s', order_book) - # top 1 = index 0 - try: - rate_from_l2 = order_book[f"{bid_strategy['price_side']}s"][order_book_top - 1][0] - except (IndexError, KeyError) as e: - logger.warning( - "Buy Price from orderbook could not be determined." - f"Orderbook: {order_book}" - ) - raise PricingError from e - logger.info(f"Buy price from orderbook {bid_strategy['price_side'].capitalize()} side " - f"- top {order_book_top} order book buy rate {rate_from_l2:.8f}") - used_rate = rate_from_l2 - else: - logger.info(f"Using Last {bid_strategy['price_side'].capitalize()} / Last Price") - ticker = self.exchange.fetch_ticker(pair) - ticker_rate = ticker[bid_strategy['price_side']] - if ticker['last'] and ticker_rate > ticker['last']: - balance = bid_strategy['ask_last_balance'] - ticker_rate = ticker_rate + balance * (ticker['last'] - ticker_rate) - used_rate = ticker_rate - - self._buy_rate_cache[pair] = used_rate - - return used_rate def create_trade(self, pair: str) -> bool: """ @@ -532,7 +487,7 @@ class FreqtradeBot(LoggingMixin): buy_limit_requested = price else: # Calculate price - buy_limit_requested = self.get_buy_rate(pair, True) + buy_limit_requested = self.exchange.get_buy_rate(pair, True) if not buy_limit_requested: raise PricingError('Could not determine buy price.') @@ -657,7 +612,7 @@ class FreqtradeBot(LoggingMixin): """ Sends rpc notification when a buy cancel occurred. """ - current_rate = self.get_buy_rate(trade.pair, False) + current_rate = self.exchange.get_buy_rate(trade.pair, False) msg = { 'trade_id': trade.id, diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index b6b395802..0e68d054d 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1684,6 +1684,50 @@ def test_fetch_l2_order_book_exception(default_conf, mocker, exchange_name): exchange.fetch_l2_order_book(pair='ETH/BTC', limit=50) +@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ + ('ask', 20, 19, 10, 0.0, 20), # Full ask side + ('ask', 20, 19, 10, 1.0, 10), # Full last side + ('ask', 20, 19, 10, 0.5, 15), # Between ask and last + ('ask', 20, 19, 10, 0.7, 13), # Between ask and last + ('ask', 20, 19, 10, 0.3, 17), # Between ask and last + ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask + ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask + ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask + ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask + ('ask', 4, 5, None, 1, 4), # last not available - uses ask + ('ask', 4, 5, None, 0, 4), # last not available - uses ask + ('bid', 21, 20, 10, 0.0, 20), # Full bid side + ('bid', 21, 20, 10, 1.0, 10), # Full last side + ('bid', 21, 20, 10, 0.5, 15), # Between bid and last + ('bid', 21, 20, 10, 0.7, 13), # Between bid and last + ('bid', 21, 20, 10, 0.3, 17), # Between bid and last + ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid + ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid + ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid + ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid + ('bid', 6, 5, None, 1, 5), # last not available - uses bid + ('bid', 6, 5, None, 0, 5), # last not available - uses bid +]) +def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, + last, last_ab, expected) -> None: + caplog.set_level(logging.DEBUG) + default_conf['bid_strategy']['ask_last_balance'] = last_ab + default_conf['bid_strategy']['price_side'] = side + exchange = get_patched_exchange(mocker, default_conf) + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': ask, 'last': last, 'bid': bid}) + + assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + assert exchange.get_buy_rate('ETH/BTC', False) == expected + assert log_has("Using cached buy rate for ETH/BTC.", caplog) + # Running a 2nd time with Refresh on! + caplog.clear() + assert exchange.get_buy_rate('ETH/BTC', True) == expected + assert not log_has("Using cached buy rate for ETH/BTC.", caplog) + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 4d9284a2f..68d861ef2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -751,49 +751,6 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert ("ETH/BTC", default_conf["timeframe"]) in refresh_mock.call_args[0][0] -@pytest.mark.parametrize("side,ask,bid,last,last_ab,expected", [ - ('ask', 20, 19, 10, 0.0, 20), # Full ask side - ('ask', 20, 19, 10, 1.0, 10), # Full last side - ('ask', 20, 19, 10, 0.5, 15), # Between ask and last - ('ask', 20, 19, 10, 0.7, 13), # Between ask and last - ('ask', 20, 19, 10, 0.3, 17), # Between ask and last - ('ask', 5, 6, 10, 1.0, 5), # last bigger than ask - ('ask', 5, 6, 10, 0.5, 5), # last bigger than ask - ('ask', 10, 20, None, 0.5, 10), # last not available - uses ask - ('ask', 4, 5, None, 0.5, 4), # last not available - uses ask - ('ask', 4, 5, None, 1, 4), # last not available - uses ask - ('ask', 4, 5, None, 0, 4), # last not available - uses ask - ('bid', 21, 20, 10, 0.0, 20), # Full bid side - ('bid', 21, 20, 10, 1.0, 10), # Full last side - ('bid', 21, 20, 10, 0.5, 15), # Between bid and last - ('bid', 21, 20, 10, 0.7, 13), # Between bid and last - ('bid', 21, 20, 10, 0.3, 17), # Between bid and last - ('bid', 6, 5, 10, 1.0, 5), # last bigger than bid - ('bid', 6, 5, 10, 0.5, 5), # last bigger than bid - ('bid', 21, 20, None, 0.5, 20), # last not available - uses bid - ('bid', 6, 5, None, 0.5, 5), # last not available - uses bid - ('bid', 6, 5, None, 1, 5), # last not available - uses bid - ('bid', 6, 5, None, 0, 5), # last not available - uses bid -]) -def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, - last, last_ab, expected) -> None: - caplog.set_level(logging.DEBUG) - default_conf['bid_strategy']['ask_last_balance'] = last_ab - default_conf['bid_strategy']['price_side'] = side - freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': ask, 'last': last, 'bid': bid}) - - assert freqtrade.get_buy_rate('ETH/BTC', True) == expected - assert not log_has("Using cached buy rate for ETH/BTC.", caplog) - - assert freqtrade.get_buy_rate('ETH/BTC', False) == expected - assert log_has("Using cached buy rate for ETH/BTC.", caplog) - # Running a 2nd time with Refresh on! - caplog.clear() - assert freqtrade.get_buy_rate('ETH/BTC', True) == expected - assert not log_has("Using cached buy rate for ETH/BTC.", caplog) - def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None: patch_RPCManager(mocker) From bd1984386e1377b3e5dd297a4da60eab6ae55988 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:39:18 +0200 Subject: [PATCH 073/127] Move get_sell_rate to exchange class --- freqtrade/exchange/exchange.py | 53 ++++++++++++++++ freqtrade/freqtradebot.py | 71 +++------------------ freqtrade/rpc/rpc.py | 8 +-- tests/exchange/test_exchange.py | 104 ++++++++++++++++++++++++++++++- tests/rpc/test_rpc.py | 6 +- tests/rpc/test_rpc_apiserver.py | 2 +- tests/test_freqtradebot.py | 105 +------------------------------- 7 files changed, 172 insertions(+), 177 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 42b518566..67676d4e0 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -88,6 +88,10 @@ class Exchange: # Cache for 10 minutes ... self._fetch_tickers_cache: TTLCache = TTLCache(maxsize=1, ttl=60 * 10) + # Cache values for 1800 to avoid frequent polling of the exchange for prices + # Caching only applies to RPC methods, so prices for open trades are still + # refreshed once every iteration. + self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) # Holds candles @@ -912,6 +916,15 @@ class Exchange: except ccxt.BaseError as e: raise OperationalException(e) from e + def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, + order_book_min: int = 1): + """ + Helper generator to query orderbook in loop (used for early sell-order placing) + """ + order_book = self.fetch_l2_order_book(pair, order_book_max) + for i in range(order_book_min, order_book_max + 1): + yield order_book[side][i - 1][0] + def get_buy_rate(self, pair: str, refresh: bool) -> float: """ Calculates bid target between current ask price and last price @@ -958,6 +971,46 @@ class Exchange: return used_rate + def get_sell_rate(self, pair: str, refresh: bool) -> float: + """ + Get sell rate - either using ticker bid or first bid based on orderbook + or remain static in any other case since it's not updating. + :param pair: Pair to get rate for + :param refresh: allow cached data + :return: Bid rate + :raises PricingError if price could not be determined. + """ + if not refresh: + rate = self._sell_rate_cache.get(pair) + # Check if cache has been invalidated + if rate: + logger.debug(f"Using cached sell rate for {pair}.") + return rate + + ask_strategy = self._config.get('ask_strategy', {}) + if ask_strategy.get('use_order_book', False): + # This code is only used for notifications, selling uses the generator directly + logger.info( + f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." + ) + try: + rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) + except (IndexError, KeyError) as e: + logger.warning("Sell Price at location from orderbook could not be determined.") + raise PricingError from e + else: + ticker = self.fetch_ticker(pair) + ticker_rate = ticker[ask_strategy['price_side']] + if ticker['last'] and ticker_rate < ticker['last']: + balance = ask_strategy.get('bid_last_balance', 0.0) + ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last']) + rate = ticker_rate + + if rate is None: + raise PricingError(f"Sell-Rate for {pair} was empty.") + self._sell_rate_cache[pair] = rate + return rate + # Fee handling @retrier diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index d15748864..8628931b6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -10,7 +10,6 @@ from threading import Lock from typing import Any, Dict, List, Optional import arrow -from cachetools import TTLCache from freqtrade import __version__, constants from freqtrade.configuration import validate_config_consistency @@ -58,11 +57,6 @@ class FreqtradeBot(LoggingMixin): # Init objects self.config = config - # Cache values for 1800 to avoid frequent polling of the exchange for prices - # Caching only applies to RPC methods, so prices for open trades are still - # refreshed once every iteration. - self._sell_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) - self.strategy: IStrategy = StrategyResolver.load_strategy(self.config) # Check config consistency here since strategies can set certain options @@ -395,7 +389,6 @@ class FreqtradeBot(LoggingMixin): return trades_created - def create_trade(self, pair: str) -> bool: """ Check the implemented trading strategy for buy signals. @@ -678,56 +671,6 @@ class FreqtradeBot(LoggingMixin): return trades_closed - def _order_book_gen(self, pair: str, side: str, order_book_max: int = 1, - order_book_min: int = 1): - """ - Helper generator to query orderbook in loop (used for early sell-order placing) - """ - order_book = self.exchange.fetch_l2_order_book(pair, order_book_max) - for i in range(order_book_min, order_book_max + 1): - yield order_book[side][i - 1][0] - - def get_sell_rate(self, pair: str, refresh: bool) -> float: - """ - Get sell rate - either using ticker bid or first bid based on orderbook - The orderbook portion is only used for rpc messaging, which would otherwise fail - for BitMex (has no bid/ask in fetch_ticker) - or remain static in any other case since it's not updating. - :param pair: Pair to get rate for - :param refresh: allow cached data - :return: Bid rate - """ - if not refresh: - rate = self._sell_rate_cache.get(pair) - # Check if cache has been invalidated - if rate: - logger.debug(f"Using cached sell rate for {pair}.") - return rate - - ask_strategy = self.config.get('ask_strategy', {}) - if ask_strategy.get('use_order_book', False): - # This code is only used for notifications, selling uses the generator directly - logger.info( - f"Getting price from order book {ask_strategy['price_side'].capitalize()} side." - ) - try: - rate = next(self._order_book_gen(pair, f"{ask_strategy['price_side']}s")) - except (IndexError, KeyError) as e: - logger.warning("Sell Price at location from orderbook could not be determined.") - raise PricingError from e - else: - ticker = self.exchange.fetch_ticker(pair) - ticker_rate = ticker[ask_strategy['price_side']] - if ticker['last'] and ticker_rate < ticker['last']: - balance = ask_strategy.get('bid_last_balance', 0.0) - ticker_rate = ticker_rate - balance * (ticker_rate - ticker['last']) - rate = ticker_rate - - if rate is None: - raise PricingError(f"Sell-Rate for {pair} was empty.") - self._sell_rate_cache[pair] = rate - return rate - def handle_trade(self, trade: Trade) -> bool: """ Sells the current pair if the threshold is reached and updates the trade record. @@ -755,9 +698,9 @@ class FreqtradeBot(LoggingMixin): logger.debug(f'Using order book between {order_book_min} and {order_book_max} ' f'for selling {trade.pair}...') - order_book = self._order_book_gen(trade.pair, f"{config_ask_strategy['price_side']}s", - order_book_min=order_book_min, - order_book_max=order_book_max) + order_book = self.exchange._order_book_gen( + trade.pair, f"{config_ask_strategy['price_side']}s", + order_book_min=order_book_min, order_book_max=order_book_max) for i in range(order_book_min, order_book_max + 1): try: sell_rate = next(order_book) @@ -770,14 +713,14 @@ class FreqtradeBot(LoggingMixin): f"{sell_rate:0.8f}") # Assign sell-rate to cache - otherwise sell-rate is never updated in the cache, # resulting in outdated RPC messages - self._sell_rate_cache[trade.pair] = sell_rate + self.exchange._sell_rate_cache[trade.pair] = sell_rate if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True else: logger.debug('checking sell') - sell_rate = self.get_sell_rate(trade.pair, True) + sell_rate = self.exchange.get_sell_rate(trade.pair, True) if self._check_and_execute_sell(trade, sell_rate, buy, sell): return True @@ -1209,7 +1152,7 @@ class FreqtradeBot(LoggingMixin): profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) # Use cached rates here - it was updated seconds ago. - current_rate = self.get_sell_rate(trade.pair, False) if not fill else None + current_rate = self.exchange.get_sell_rate(trade.pair, False) if not fill else None profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" @@ -1254,7 +1197,7 @@ class FreqtradeBot(LoggingMixin): profit_rate = trade.close_rate if trade.close_rate else trade.close_rate_requested profit_trade = trade.calc_profit(rate=profit_rate) - current_rate = self.get_sell_rate(trade.pair, False) + current_rate = self.exchange.get_sell_rate(trade.pair, False) profit_ratio = trade.calc_profit_ratio(profit_rate) gain = "profit" if profit_ratio > 0 else "loss" diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index c609bccb8..40d5eb583 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -171,7 +171,7 @@ class RPC: # calculate profit and send message to user if trade.is_open: try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (ExchangeError, PricingError): current_rate = NAN else: @@ -230,7 +230,7 @@ class RPC: for trade in trades: # calculate profit and send message to user try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (PricingError, ExchangeError): current_rate = NAN trade_percent = (100 * trade.calc_profit_ratio(current_rate)) @@ -386,7 +386,7 @@ class RPC: else: # Get current rate try: - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) except (PricingError, ExchangeError): current_rate = NAN profit_ratio = trade.calc_profit_ratio(rate=current_rate) @@ -556,7 +556,7 @@ class RPC: if not fully_canceled: # Get current rate and execute sell - current_rate = self._freqtrade.get_sell_rate(trade.pair, False) + current_rate = self._freqtrade.exchange.get_sell_rate(trade.pair, False) sell_reason = SellCheckTuple(sell_type=SellType.FORCE_SELL) self._freqtrade.execute_sell(trade, current_rate, sell_reason) # ---- EOF def _exec_forcesell ---- diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 0e68d054d..5fa94e6c1 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -11,7 +11,7 @@ import pytest from pandas import DataFrame from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException, - OperationalException, TemporaryError) + OperationalException, PricingError, TemporaryError) from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken from freqtrade.exchange.common import (API_FETCH_ORDER_RETRY_COUNT, API_RETRY_COUNT, calculate_backoff) @@ -1728,6 +1728,108 @@ def test_get_buy_rate(mocker, default_conf, caplog, side, ask, bid, assert not log_has("Using cached buy rate for ETH/BTC.", caplog) +@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', [ + ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side + ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side + ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat + ('bid', 12.0, 11.2, 10.5, 0.0, 11.2), # Last smaller than bid + ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid + ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid + ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), + ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side + ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side + ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat + ('ask', 12.2, 11.2, 10.5, 0.0, 12.2), # Last smaller than ask + ('ask', 12.0, 11.0, 10.5, 1.0, 12.0), # Last smaller than ask - uses ask + ('ask', 12.0, 11.2, 10.5, 0.5, 12.0), # Last smaller than ask - uses ask + ('ask', 10.0, 11.0, 11.0, 0.0, 10.0), + ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), + ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), + ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), +]) +def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, + last, last_ab, expected) -> None: + caplog.set_level(logging.DEBUG) + + default_conf['ask_strategy']['price_side'] = side + default_conf['ask_strategy']['bid_last_balance'] = last_ab + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': ask, 'bid': bid, 'last': last}) + pair = "ETH/BTC" + + # Test regular mode + exchange = get_patched_exchange(mocker, default_conf) + rate = exchange.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) + assert isinstance(rate, float) + assert rate == expected + # Use caching + rate = exchange.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + +@pytest.mark.parametrize('side,expected', [ + ('bid', 0.043936), # Value from order_book_l2 fiture - bids side + ('ask', 0.043949), # Value from order_book_l2 fiture - asks side +]) +def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2): + caplog.set_level(logging.DEBUG) + # Test orderbook mode + default_conf['ask_strategy']['price_side'] = side + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) + exchange = get_patched_exchange(mocker, default_conf) + rate = exchange.get_sell_rate(pair, True) + assert not log_has("Using cached sell rate for ETH/BTC.", caplog) + assert isinstance(rate, float) + assert rate == expected + rate = exchange.get_sell_rate(pair, False) + assert rate == expected + assert log_has("Using cached sell rate for ETH/BTC.", caplog) + + +def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog): + # Test orderbook mode + default_conf['ask_strategy']['price_side'] = 'ask' + default_conf['ask_strategy']['use_order_book'] = True + default_conf['ask_strategy']['order_book_min'] = 1 + default_conf['ask_strategy']['order_book_max'] = 2 + pair = "ETH/BTC" + # Test What happens if the exchange returns an empty orderbook. + mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', + return_value={'bids': [[]], 'asks': [[]]}) + exchange = get_patched_exchange(mocker, default_conf) + with pytest.raises(PricingError): + exchange.get_sell_rate(pair, True) + assert log_has("Sell Price at location from orderbook could not be determined.", caplog) + + +def test_get_sell_rate_exception(default_conf, mocker, caplog): + # Ticker on one side can be empty in certain circumstances. + default_conf['ask_strategy']['price_side'] = 'ask' + pair = "ETH/BTC" + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': None, 'bid': 0.12, 'last': None}) + exchange = get_patched_exchange(mocker, default_conf) + with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): + exchange.get_sell_rate(pair, True) + + exchange._config['ask_strategy']['price_side'] = 'bid' + assert exchange.get_sell_rate(pair, True) == 0.12 + # Reverse sides + mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', + return_value={'ask': 0.13, 'bid': None, 'last': None}) + with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): + exchange.get_sell_rate(pair, True) + + exchange._config['ask_strategy']['price_side'] = 'ask' + assert exchange.get_sell_rate(pair, True) == 0.13 + + def make_fetch_ohlcv_mock(data): def fetch_ohlcv_mock(pair, timeframe, since): if since: diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index e7a968e37..7556dde6d 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -109,7 +109,7 @@ def test_rpc_trade_status(default_conf, ticker, fee, mocker) -> None: 'exchange': 'binance', } - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) results = rpc._rpc_trade_status() assert isnan(results[0]['current_profit']) @@ -217,7 +217,7 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None: assert '-0.41% (-0.06)' == result[0][3] assert '-0.06' == f'{fiat_profit_sum:.2f}' - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) result, headers, fiat_profit_sum = rpc._rpc_status_table(default_conf['stake_currency'], 'USD') assert 'instantly' == result[0][2] @@ -427,7 +427,7 @@ def test_rpc_trade_statistics(default_conf, ticker, ticker_sell_up, fee, assert prec_satoshi(stats['best_rate'], 6.2) # Test non-available pair - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) stats = rpc._rpc_trade_statistics(stake_currency, fiat_display_currency) assert stats['trade_count'] == 2 diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 1a66b2e81..def2e43c6 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -834,7 +834,7 @@ def test_api_status(botclient, mocker, ticker, fee, markets): 'exchange': 'binance', } - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', MagicMock(side_effect=ExchangeError("Pair 'ETH/BTC' not available"))) rc = client_get(client, f"{BASE_URI}/status") diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 68d861ef2..9039328b0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -751,7 +751,6 @@ def test_process_informative_pairs_added(default_conf, ticker, mocker) -> None: assert ("ETH/BTC", default_conf["timeframe"]) in refresh_mock.call_args[0][0] - def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order_open) -> None: patch_RPCManager(mocker) patch_exchange(mocker) @@ -2480,7 +2479,7 @@ def test_handle_cancel_sell_limit(mocker, default_conf, fee) -> None: 'freqtrade.exchange.Exchange', cancel_order=cancel_order_mock, ) - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_sell_rate', return_value=0.245441) + mocker.patch('freqtrade.exchange.Exchange.get_sell_rate', return_value=0.245441) freqtrade = FreqtradeBot(default_conf) @@ -4029,108 +4028,6 @@ def test_order_book_ask_strategy(default_conf, limit_buy_order_open, limit_buy_o assert log_has('Sell Price at location 1 from orderbook could not be determined.', caplog) -@pytest.mark.parametrize('side,ask,bid,last,last_ab,expected', [ - ('bid', 12.0, 11.0, 11.5, 0.0, 11.0), # full bid side - ('bid', 12.0, 11.0, 11.5, 1.0, 11.5), # full last side - ('bid', 12.0, 11.0, 11.5, 0.5, 11.25), # between bid and lat - ('bid', 12.0, 11.2, 10.5, 0.0, 11.2), # Last smaller than bid - ('bid', 12.0, 11.2, 10.5, 1.0, 11.2), # Last smaller than bid - uses bid - ('bid', 12.0, 11.2, 10.5, 0.5, 11.2), # Last smaller than bid - uses bid - ('bid', 0.003, 0.002, 0.005, 0.0, 0.002), - ('ask', 12.0, 11.0, 12.5, 0.0, 12.0), # full ask side - ('ask', 12.0, 11.0, 12.5, 1.0, 12.5), # full last side - ('ask', 12.0, 11.0, 12.5, 0.5, 12.25), # between bid and lat - ('ask', 12.2, 11.2, 10.5, 0.0, 12.2), # Last smaller than ask - ('ask', 12.0, 11.0, 10.5, 1.0, 12.0), # Last smaller than ask - uses ask - ('ask', 12.0, 11.2, 10.5, 0.5, 12.0), # Last smaller than ask - uses ask - ('ask', 10.0, 11.0, 11.0, 0.0, 10.0), - ('ask', 10.11, 11.2, 11.0, 0.0, 10.11), - ('ask', 0.001, 0.002, 11.0, 0.0, 0.001), - ('ask', 0.006, 1.0, 11.0, 0.0, 0.006), -]) -def test_get_sell_rate(default_conf, mocker, caplog, side, bid, ask, - last, last_ab, expected) -> None: - caplog.set_level(logging.DEBUG) - - default_conf['ask_strategy']['price_side'] = side - default_conf['ask_strategy']['bid_last_balance'] = last_ab - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': ask, 'bid': bid, 'last': last}) - pair = "ETH/BTC" - - # Test regular mode - ft = get_patched_freqtradebot(mocker, default_conf) - rate = ft.get_sell_rate(pair, True) - assert not log_has("Using cached sell rate for ETH/BTC.", caplog) - assert isinstance(rate, float) - assert rate == expected - # Use caching - rate = ft.get_sell_rate(pair, False) - assert rate == expected - assert log_has("Using cached sell rate for ETH/BTC.", caplog) - - -@pytest.mark.parametrize('side,expected', [ - ('bid', 0.043936), # Value from order_book_l2 fiture - bids side - ('ask', 0.043949), # Value from order_book_l2 fiture - asks side -]) -def test_get_sell_rate_orderbook(default_conf, mocker, caplog, side, expected, order_book_l2): - caplog.set_level(logging.DEBUG) - # Test orderbook mode - default_conf['ask_strategy']['price_side'] = side - default_conf['ask_strategy']['use_order_book'] = True - default_conf['ask_strategy']['order_book_min'] = 1 - default_conf['ask_strategy']['order_book_max'] = 2 - pair = "ETH/BTC" - mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', order_book_l2) - ft = get_patched_freqtradebot(mocker, default_conf) - rate = ft.get_sell_rate(pair, True) - assert not log_has("Using cached sell rate for ETH/BTC.", caplog) - assert isinstance(rate, float) - assert rate == expected - rate = ft.get_sell_rate(pair, False) - assert rate == expected - assert log_has("Using cached sell rate for ETH/BTC.", caplog) - - -def test_get_sell_rate_orderbook_exception(default_conf, mocker, caplog): - # Test orderbook mode - default_conf['ask_strategy']['price_side'] = 'ask' - default_conf['ask_strategy']['use_order_book'] = True - default_conf['ask_strategy']['order_book_min'] = 1 - default_conf['ask_strategy']['order_book_max'] = 2 - pair = "ETH/BTC" - # Test What happens if the exchange returns an empty orderbook. - mocker.patch('freqtrade.exchange.Exchange.fetch_l2_order_book', - return_value={'bids': [[]], 'asks': [[]]}) - ft = get_patched_freqtradebot(mocker, default_conf) - with pytest.raises(PricingError): - ft.get_sell_rate(pair, True) - assert log_has("Sell Price at location from orderbook could not be determined.", caplog) - - -def test_get_sell_rate_exception(default_conf, mocker, caplog): - # Ticker on one side can be empty in certain circumstances. - default_conf['ask_strategy']['price_side'] = 'ask' - pair = "ETH/BTC" - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': None, 'bid': 0.12, 'last': None}) - ft = get_patched_freqtradebot(mocker, default_conf) - with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): - ft.get_sell_rate(pair, True) - - ft.config['ask_strategy']['price_side'] = 'bid' - assert ft.get_sell_rate(pair, True) == 0.12 - # Reverse sides - mocker.patch('freqtrade.exchange.Exchange.fetch_ticker', - return_value={'ask': 0.13, 'bid': None, 'last': None}) - with pytest.raises(PricingError, match=r"Sell-Rate for ETH/BTC was empty."): - ft.get_sell_rate(pair, True) - - ft.config['ask_strategy']['price_side'] = 'ask' - assert ft.get_sell_rate(pair, True) == 0.13 - - def test_startup_state(default_conf, mocker): default_conf['pairlist'] = {'method': 'VolumePairList', 'config': {'number_assets': 20} From 387f3bbc5d06b1123e03b586c7fcad4cce9d3868 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 2 Jun 2021 11:43:47 +0200 Subject: [PATCH 074/127] Adjust missed tests --- tests/test_freqtradebot.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 9039328b0..66866a8fc 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -759,13 +759,10 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order stake_amount = 2 bid = 0.11 buy_rate_mock = MagicMock(return_value=bid) - mocker.patch.multiple( - 'freqtrade.freqtradebot.FreqtradeBot', - get_buy_rate=buy_rate_mock, - ) buy_mm = MagicMock(return_value=limit_buy_order_open) mocker.patch.multiple( 'freqtrade.exchange.Exchange', + get_buy_rate=buy_rate_mock, fetch_ticker=MagicMock(return_value={ 'bid': 0.00001172, 'ask': 0.00001173, @@ -856,7 +853,7 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order assert not freqtrade.execute_buy(pair, stake_amount) # Fail to get price... - mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_buy_rate', MagicMock(return_value=0.0)) + mocker.patch('freqtrade.exchange.Exchange.get_buy_rate', MagicMock(return_value=0.0)) with pytest.raises(PricingError, match="Could not determine buy price."): freqtrade.execute_buy(pair, stake_amount) @@ -864,10 +861,6 @@ def test_execute_buy(mocker, default_conf, fee, limit_buy_order, limit_buy_order def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) -> None: freqtrade = get_patched_freqtradebot(mocker, default_conf) - mocker.patch.multiple( - 'freqtrade.freqtradebot.FreqtradeBot', - get_buy_rate=MagicMock(return_value=0.11), - ) mocker.patch.multiple( 'freqtrade.exchange.Exchange', fetch_ticker=MagicMock(return_value={ @@ -876,6 +869,7 @@ def test_execute_buy_confirm_error(mocker, default_conf, fee, limit_buy_order) - 'last': 0.00001172 }), buy=MagicMock(return_value=limit_buy_order), + get_buy_rate=MagicMock(return_value=0.11), get_min_pair_stake_amount=MagicMock(return_value=1), get_fee=fee, ) @@ -3934,7 +3928,7 @@ def test_order_book_bid_strategy1(mocker, default_conf, order_book_l2) -> None: default_conf['telegram']['enabled'] = False freqtrade = FreqtradeBot(default_conf) - assert freqtrade.get_buy_rate('ETH/BTC', True) == 0.043935 + assert freqtrade.exchange.get_buy_rate('ETH/BTC', True) == 0.043935 assert ticker_mock.call_count == 0 @@ -3956,7 +3950,7 @@ def test_order_book_bid_strategy_exception(mocker, default_conf, caplog) -> None freqtrade = FreqtradeBot(default_conf) # orderbook shall be used even if tickers would be lower. with pytest.raises(PricingError): - freqtrade.get_buy_rate('ETH/BTC', refresh=True) + freqtrade.exchange.get_buy_rate('ETH/BTC', refresh=True) assert log_has_re(r'Buy Price from orderbook could not be determined.', caplog) From cabab44b759934b528fb301023f7d17330131b43 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 30 May 2021 19:15:59 +0200 Subject: [PATCH 075/127] Combine docker build scripts --- .github/workflows/ci.yml | 14 ++--- .travis.yml | 6 -- ...lish_docker.sh => publish_docker_multi.sh} | 63 +++++++++++++------ build_helpers/publish_docker_pi.sh | 49 --------------- docker/Dockerfile.armhf | 6 +- 5 files changed, 52 insertions(+), 86 deletions(-) rename build_helpers/{publish_docker.sh => publish_docker_multi.sh} (54%) delete mode 100755 build_helpers/publish_docker_pi.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea766d77d..1ec0f22ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,7 @@ on: - master - stable - develop + - test_multiarch tags: release: types: [published] @@ -75,7 +76,7 @@ jobs: COVERALLS_REPO_TOKEN: 6D1m0xupS3FgutfuGao8keFf9Hc0FpIXu run: | # Allow failure for coveralls - coveralls -v || true + coveralls || true - name: Backtesting run: | @@ -392,19 +393,12 @@ jobs: - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} - - name: Build and test and push docker image + - name: Build and test and push docker images env: IMAGE_NAME: freqtradeorg/freqtrade BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} run: | - build_helpers/publish_docker.sh - - - name: Build Raspberry docker image - env: - IMAGE_NAME: freqtradeorg/freqtrade - BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }} - run: | - build_helpers/publish_docker_pi.sh + build_helpers/publish_docker_multi.sh - name: Slack Notification diff --git a/.travis.yml b/.travis.yml index 03a8df49b..4535c44cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,12 +46,6 @@ jobs: - script: mypy freqtrade scripts name: mypy - # - stage: docker - # if: branch in (master, develop, feat/improve_travis) AND (type in (push, cron)) - # script: - # - build_helpers/publish_docker.sh - # name: "Build and test and push docker image" - notifications: slack: secure: bKLXmOrx8e2aPZl7W8DA5BdPAXWGpI5UzST33oc1G/thegXcDVmHBTJrBs4sZak6bgAclQQrdZIsRd2eFYzHLalJEaw6pk7hoAw8SvLnZO0ZurWboz7qg2+aZZXfK4eKl/VUe4sM9M4e/qxjkK+yWG7Marg69c4v1ypF7ezUi1fPYILYw8u0paaiX0N5UX8XNlXy+PBlga2MxDjUY70MuajSZhPsY2pDUvYnMY1D/7XN3cFW0g+3O8zXjF0IF4q1Z/1ASQe+eYjKwPQacE+O8KDD+ZJYoTOFBAPllrtpO1jnOPFjNGf3JIbVMZw4bFjIL0mSQaiSUaUErbU3sFZ5Or79rF93XZ81V7uEZ55vD8KMfR2CB1cQJcZcj0v50BxLo0InkFqa0Y8Nra3sbpV4fV5Oe8pDmomPJrNFJnX6ULQhQ1gTCe0M5beKgVms5SITEpt4/Y0CmLUr6iHDT0CUiyMIRWAXdIgbGh1jfaWOMksybeRevlgDsIsNBjXmYI1Sw2ZZR2Eo2u4R6zyfyjOMLwYJ3vgq9IrACv2w5nmf0+oguMWHf6iWi2hiOqhlAN1W74+3HsYQcqnuM3LGOmuCnPprV1oGBqkPXjIFGpy21gNx4vHfO1noLUyJnMnlu2L7SSuN1CdLsnjJ1hVjpJjPfqB4nn8g12x87TqM1bOm+3Q= diff --git a/build_helpers/publish_docker.sh b/build_helpers/publish_docker_multi.sh similarity index 54% rename from build_helpers/publish_docker.sh rename to build_helpers/publish_docker_multi.sh index da9fc4e34..a6b06ce7d 100755 --- a/build_helpers/publish_docker.sh +++ b/build_helpers/publish_docker_multi.sh @@ -1,21 +1,48 @@ #!/bin/sh +# The below assumes a correctly setup docker buildx environment + # Replace / with _ to create a valid tag TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") TAG_PLOT=${TAG}_plot +TAG_PI="${TAG}_pi" + +PI_PLATFORM="linux/arm/v7" echo "Running for ${TAG}" +CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache # Add commit and commit_message to docker container echo "${GITHUB_SHA}" > freqtrade_commit if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" + # Build regular image docker build -t freqtrade:${TAG} . + # Build PI image + docker buildx build \ + --cache-to=type=registry,ref=${CACHE_TAG} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} --push . else echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Pull last build to avoid rebuilding the whole image + # Build regular image docker pull ${IMAGE_NAME}:${TAG} docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . + + # Pull last build to avoid rebuilding the whole image + # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} + docker buildx build \ + --cache-from=type=registry,ref=${CACHE_TAG} \ + --cache-to=type=registry,ref=${CACHE_TAG} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} --push . +fi + +if [ $? -ne 0 ]; then + echo "failed building multiarch images" + return 1 fi # Tag image for upload and next build step docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG @@ -24,11 +51,6 @@ docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t fre docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi - # Run backtest docker run --rm -v $(pwd)/config_bittrex.json.example:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy DefaultStrategy @@ -37,24 +59,29 @@ if [ $? -ne 0 ]; then return 1 fi -if [ $? -ne 0 ]; then - echo "failed tagging image" - return 1 -fi - -# Tag as latest for develop builds -if [ "${TAG}" = "develop" ]; then - docker tag freqtrade:$TAG ${IMAGE_NAME}:latest -fi - -# Show all available images docker images docker push ${IMAGE_NAME} docker push ${IMAGE_NAME}:$TAG_PLOT docker push ${IMAGE_NAME}:$TAG +# Create multiarch image +# Make sure that all images contained here are pushed to github first. +# Otherwise installation might fail. + +docker manifest create freqtradeorg/freqtrade:${TAG} ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} +docker manifest push freqtradeorg/freqtrade:${TAG} + +# Tag as latest for develop builds +if [ "${TAG}" = "develop" ]; then + docker manifest create freqtradeorg/freqtrade:latest ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG_PI} + docker manifest push freqtradeorg/freqtrade:latest +fi + + +docker images + if [ $? -ne 0 ]; then - echo "failed pushing repo" + echo "failed building image" return 1 fi diff --git a/build_helpers/publish_docker_pi.sh b/build_helpers/publish_docker_pi.sh deleted file mode 100755 index c7024828b..000000000 --- a/build_helpers/publish_docker_pi.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh - -# The below assumes a correctly setup docker buildx environment - -# Replace / with _ to create a valid tag -TAG_ORIG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") -TAG="${TAG_ORIG}_pi" - -PI_PLATFORM="linux/arm/v7" -echo "Running for ${TAG}" -CACHE_TAG=freqtradeorg/freqtrade_cache:${TAG}_cache - -# Add commit and commit_message to docker container -echo "${GITHUB_SHA}" > freqtrade_commit - -if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then - echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" - docker buildx build \ - --cache-to=type=registry,ref=${CACHE_TAG} \ - -f docker/Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG} --push . -else - echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Pull last build to avoid rebuilding the whole image - # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} - docker buildx build \ - --cache-from=type=registry,ref=${CACHE_TAG} \ - --cache-to=type=registry,ref=${CACHE_TAG} \ - -f docker/Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG} --push . -fi - -docker images - -# Create multiarch image -# Make sure that all images contained here are pushed to github first. -# Otherwise installation might fail. - -docker manifest create freqtradeorg/freqtrade:${TAG_ORIG} ${IMAGE_NAME}:${TAG_ORIG} ${IMAGE_NAME}:${TAG} -docker manifest push freqtradeorg/freqtrade:${TAG_ORIG} - -docker images - -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index 8abf0e44b..f9827774e 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM --platform=linux/arm/v7 python:3.7.10-slim-buster as base +FROM python:3.7.10-slim-buster as base # Setup env ENV LANG C.UTF-8 @@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker" # Prepare environment RUN mkdir /freqtrade \ && apt-get update \ - && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-serial-dev \ + && apt-get -y install sudo libatlas3-base curl sqlite3 libhdf5-dev \ && apt-get clean \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ @@ -23,7 +23,7 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps RUN apt-get update \ - && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 \ + && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 pkg-config cmake gcc \ && apt-get clean \ && pip install --upgrade pip \ && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf From 80af6e43e4f94cc442f8341a0ed1c1b7e691ff22 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 2 Jun 2021 20:50:08 +0200 Subject: [PATCH 076/127] test-pairlist: remove non-JSON headline from JSON output --- freqtrade/commands/pairlist_commands.py | 2 +- tests/commands/test_commands.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index 0661cd03c..a2b10a42f 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -31,7 +31,7 @@ def start_test_pairlist(args: Dict[str, Any]) -> None: results[curr] = pairlists.whitelist for curr, pairlist in results.items(): - if not args.get('print_one_column', False): + if not args.get('print_one_column', False) and not args.get('list_pairs_print_json', False): print(f"Pairs for {curr}: ") if args.get('print_one_column', False): diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 71ae0ed78..b0808ae06 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1,3 +1,4 @@ +import json import re from io import BytesIO from pathlib import Path @@ -914,8 +915,15 @@ def test_start_test_pairlist(mocker, caplog, tickers, default_conf, capsys): ] start_test_pairlist(get_args(args)) captured = capsys.readouterr() - assert re.match(r'Pairs for BTC: \n\["ETH/BTC","TKN/BTC","BLK/BTC","LTC/BTC","XRP/BTC"\]\n', - captured.out) + try: + json_pairs = json.loads(captured.out) + assert 'ETH/BTC' in json_pairs + assert 'TKN/BTC' in json_pairs + assert 'BLK/BTC' in json_pairs + assert 'LTC/BTC' in json_pairs + assert 'XRP/BTC' in json_pairs + except json.decoder.JSONDecodeError: + pytest.fail(f'Expected well formed JSON, but failed to parse: {captured.out}') def test_hyperopt_list(mocker, capsys, caplog, saved_hyperopt_results, From 812eb229df04db2461fdccc49d2f577bf64f8ab2 Mon Sep 17 00:00:00 2001 From: Janos Date: Sun, 30 May 2021 16:11:24 +0200 Subject: [PATCH 077/127] plot-profit: Make "auto-open" HTML result optional Adding an "--auto-open" argument. This improves tool processing of the results, while still allowing to open the HTML file for easy use. --- docs/plotting.md | 1 + freqtrade/commands/arguments.py | 2 +- freqtrade/commands/cli_options.py | 5 +++++ freqtrade/configuration/configuration.py | 3 +++ freqtrade/plot/plotting.py | 3 ++- tests/test_arguments.py | 6 +++++- tests/test_plotting.py | 2 +- 7 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/plotting.md b/docs/plotting.md index 05708ce66..9fae38504 100644 --- a/docs/plotting.md +++ b/docs/plotting.md @@ -275,6 +275,7 @@ optional arguments: (backtest file)) Default: file -i TIMEFRAME, --timeframe TIMEFRAME, --ticker-interval TIMEFRAME Specify timeframe (`1m`, `5m`, `30m`, `1h`, `1d`). + --auto-open Automatically open generated plot. Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index ffd317799..7f4f7edd6 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -69,7 +69,7 @@ ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "timerange", "timeframe", "no_trades"] ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url", - "trade_source", "timeframe"] + "trade_source", "timeframe", "plot_auto_open"] ARGS_INSTALL_UI = ["erase_ui_only"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index b583b47ba..d832693ee 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -433,6 +433,11 @@ AVAILABLE_CLI_OPTIONS = { metavar='INT', default=750, ), + "plot_auto_open": Arg( + '--auto-open', + help='Automatically open generated plot.', + action='store_true', + ), "no_trades": Arg( '--no-trades', help='Skip using trades from backtesting file and DB.', diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index f6d0520c5..abb9f994b 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -375,6 +375,9 @@ class Configuration: self._args_to_config(config, argname='plot_limit', logstring='Limiting plot to: {}') + self._args_to_config(config, argname='plot_auto_open', + logstring='Parameter --auto-open detected.') + self._args_to_config(config, argname='trade_source', logstring='Using trades from: {}') diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index b62ae6015..f52b0bc81 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -602,4 +602,5 @@ def plot_profit(config: Dict[str, Any]) -> None: trades, config.get('timeframe', '5m'), config.get('stake_currency', '')) store_plot_file(fig, filename='freqtrade-profit-plot.html', - directory=config['user_data_dir'] / 'plot', auto_open=True) + directory=config['user_data_dir'] / 'plot', + auto_open=config.get('plot_auto_open', False)) diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 60c2cfbac..0d81dea28 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -186,18 +186,22 @@ def test_plot_dataframe_options() -> None: assert pargs['pairs'] == ['UNITTEST/BTC'] -def test_plot_profit_options() -> None: +@pytest.mark.parametrize('auto_open_arg', [True, False]) +def test_plot_profit_options(auto_open_arg: bool) -> None: args = [ 'plot-profit', '-p', 'UNITTEST/BTC', '--trade-source', 'DB', '--db-url', 'sqlite:///whatever.sqlite', ] + if auto_open_arg: + args.append('--auto-open') pargs = Arguments(args).get_parsed_arg() assert pargs['trade_source'] == 'DB' assert pargs['pairs'] == ['UNITTEST/BTC'] assert pargs['db_url'] == 'sqlite:///whatever.sqlite' + assert pargs['plot_auto_open'] == auto_open_arg def test_config_notallowed(mocker) -> None: diff --git a/tests/test_plotting.py b/tests/test_plotting.py index a22c8c681..0b1054ced 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -460,7 +460,7 @@ def test_plot_profit(default_conf, mocker, testdatadir): assert store_mock.call_count == 1 assert profit_mock.call_args_list[0][0][0] == default_conf['pairs'] - assert store_mock.call_args_list[0][1]['auto_open'] is True + assert store_mock.call_args_list[0][1]['auto_open'] is False @pytest.mark.parametrize("ind1,ind2,plot_conf,exp", [ From 42b6d28b3ca0bab08a4d37e5cafa302d8d86e69f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 3 Jun 2021 19:20:04 +0200 Subject: [PATCH 078/127] Update warning about order_time_in_force as pointed out in #3009 --- docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7fa751bef..ef6f34094 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -406,8 +406,8 @@ The possible values are: `gtc` (default), `fok` or `ioc`. ``` !!! Warning - This is an ongoing work. For now it is supported only for binance and only for buy orders. - Please don't change the default value unless you know what you are doing. + This is an ongoing work. For now it is supported only for binance. + Please don't change the default value unless you know what you are doing and have researched the impact of using different values. ### Exchange configuration From a0893b291a099b928ced567b9ef81f2d2795717a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 5 Jun 2021 09:03:03 +0200 Subject: [PATCH 079/127] Fix strategy samples to use runmode.value closes #5073 --- freqtrade/templates/sample_strategy.py | 2 +- freqtrade/templates/subtemplates/indicators_full.j2 | 2 +- freqtrade/templates/subtemplates/indicators_minimal.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/freqtrade/templates/sample_strategy.py b/freqtrade/templates/sample_strategy.py index e51feff1e..282b2f8e2 100644 --- a/freqtrade/templates/sample_strategy.py +++ b/freqtrade/templates/sample_strategy.py @@ -329,7 +329,7 @@ class SampleStrategy(IStrategy): """ # first check if dataprovider is available if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] diff --git a/freqtrade/templates/subtemplates/indicators_full.j2 b/freqtrade/templates/subtemplates/indicators_full.j2 index 57d2ca665..a497b47cb 100644 --- a/freqtrade/templates/subtemplates/indicators_full.j2 +++ b/freqtrade/templates/subtemplates/indicators_full.j2 @@ -199,7 +199,7 @@ dataframe['htleadsine'] = hilbert['leadsine'] """ # first check if dataprovider is available if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] diff --git a/freqtrade/templates/subtemplates/indicators_minimal.j2 b/freqtrade/templates/subtemplates/indicators_minimal.j2 index 7d75b4610..90f4f4d4a 100644 --- a/freqtrade/templates/subtemplates/indicators_minimal.j2 +++ b/freqtrade/templates/subtemplates/indicators_minimal.j2 @@ -10,7 +10,7 @@ dataframe['rsi'] = ta.RSI(dataframe) """ # first check if dataprovider is available if self.dp: - if self.dp.runmode in ('live', 'dry_run'): + if self.dp.runmode.value in ('live', 'dry_run'): ob = self.dp.orderbook(metadata['pair'], 1) dataframe['best_bid'] = ob['bids'][0][0] dataframe['best_ask'] = ob['asks'][0][0] From 6479217cb485e39dcee1142c707ebf8436e9f786 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 6 Jun 2021 14:16:32 +0200 Subject: [PATCH 080/127] Don't build for test_multiarch --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ec0f22ae..42959c3b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,6 @@ on: - master - stable - develop - - test_multiarch tags: release: types: [published] From c8accd314a0aa8fce5c3edb67dc89c2782dc8e4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jun 2021 05:21:47 +0000 Subject: [PATCH 081/127] Bump uvicorn from 0.13.4 to 0.14.0 Bumps [uvicorn](https://github.com/encode/uvicorn) from 0.13.4 to 0.14.0. - [Release notes](https://github.com/encode/uvicorn/releases) - [Changelog](https://github.com/encode/uvicorn/blob/master/CHANGELOG.md) - [Commits](https://github.com/encode/uvicorn/compare/0.13.4...0.14.0) --- updated-dependencies: - dependency-name: uvicorn dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8f4fed9ad..3ee290d2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ sdnotify==0.3.2 # API Server fastapi==0.65.1 -uvicorn==0.13.4 +uvicorn==0.14.0 pyjwt==2.1.0 aiofiles==0.7.0 From 9073a053282b526954b6732bbcf484206be21d64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jun 2021 05:21:53 +0000 Subject: [PATCH 082/127] Bump pytest-cov from 2.12.0 to 2.12.1 Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 2.12.0 to 2.12.1. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.12.0...v2.12.1) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 7b106ae9b..6fbe581a5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -10,7 +10,7 @@ flake8-tidy-imports==4.3.0 mypy==0.812 pytest==6.2.4 pytest-asyncio==0.15.1 -pytest-cov==2.12.0 +pytest-cov==2.12.1 pytest-mock==3.6.1 pytest-random-order==1.0.4 isort==5.8.0 From 69d74544aafafd5c48fc21913bb04243072abd27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jun 2021 05:22:00 +0000 Subject: [PATCH 083/127] Bump python-telegram-bot from 13.5 to 13.6 Bumps [python-telegram-bot](https://github.com/python-telegram-bot/python-telegram-bot) from 13.5 to 13.6. - [Release notes](https://github.com/python-telegram-bot/python-telegram-bot/releases) - [Changelog](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/CHANGES.rst) - [Commits](https://github.com/python-telegram-bot/python-telegram-bot/compare/v13.5...v13.6) --- updated-dependencies: - dependency-name: python-telegram-bot dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8f4fed9ad..a232de96b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==1.50.70 cryptography==3.4.7 aiohttp==3.7.4.post0 SQLAlchemy==1.4.17 -python-telegram-bot==13.5 +python-telegram-bot==13.6 arrow==1.1.0 cachetools==4.2.2 requests==2.25.1 From 2468ae35cda1d3a4e56a2458008de01e86e84d23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jun 2021 05:22:09 +0000 Subject: [PATCH 084/127] Bump ccxt from 1.50.70 to 1.51.3 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.50.70 to 1.51.3. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.50.70...1.51.3) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8f4fed9ad..e10494b2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.20.3 pandas==1.2.4 -ccxt==1.50.70 +ccxt==1.51.3 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From 77a2feeb9f7a1ea0c1153512297a43fb5bf2c1dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jun 2021 05:22:24 +0000 Subject: [PATCH 085/127] Bump pycoingecko from 2.0.0 to 2.1.0 Bumps [pycoingecko](https://github.com/man-c/pycoingecko) from 2.0.0 to 2.1.0. - [Release notes](https://github.com/man-c/pycoingecko/releases) - [Changelog](https://github.com/man-c/pycoingecko/blob/master/CHANGELOG.md) - [Commits](https://github.com/man-c/pycoingecko/compare/2.0.0...2.1.0) --- updated-dependencies: - dependency-name: pycoingecko dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8f4fed9ad..5a857b605 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ jsonschema==3.2.0 TA-Lib==0.4.20 technical==1.3.0 tabulate==0.8.9 -pycoingecko==2.0.0 +pycoingecko==2.1.0 jinja2==3.0.1 tables==3.6.1 blosc==1.10.2 From 14119d7366eabdd6ea4f56074590bacff0050143 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jun 2021 05:22:30 +0000 Subject: [PATCH 086/127] Bump mkdocs-material from 7.1.6 to 7.1.7 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.1.6 to 7.1.7. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.1.6...7.1.7) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 3f8020e8c..ff07905b7 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,3 @@ -mkdocs-material==7.1.6 +mkdocs-material==7.1.7 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From 57cd8888e21ca457a8c3a408e9c3f838f228ea8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jun 2021 17:34:25 +0000 Subject: [PATCH 087/127] Bump blosc from 1.10.2 to 1.10.4 Bumps [blosc](https://github.com/blosc/python-blosc) from 1.10.2 to 1.10.4. - [Release notes](https://github.com/blosc/python-blosc/releases) - [Changelog](https://github.com/Blosc/python-blosc/blob/master/RELEASE_NOTES.rst) - [Commits](https://github.com/blosc/python-blosc/compare/v1.10.2...v1.10.4) --- updated-dependencies: - dependency-name: blosc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6b7331e20..0e4d35464 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ tabulate==0.8.9 pycoingecko==2.1.0 jinja2==3.0.1 tables==3.6.1 -blosc==1.10.2 +blosc==1.10.4 # find first, C search in arrays py_find_1st==1.1.5 From dff8490daa14b44c4f7a4bfc57e7eadbf20b59e9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Jun 2021 20:00:30 +0200 Subject: [PATCH 088/127] Fix docs rendering for pricefilter --- docs/includes/pairlists.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/includes/pairlists.md b/docs/includes/pairlists.md index ce0cc6e57..f19c5a181 100644 --- a/docs/includes/pairlists.md +++ b/docs/includes/pairlists.md @@ -122,8 +122,8 @@ The `max_price` setting removes pairs where the price is above the specified pri This option is disabled by default, and will only apply if set to > 0. The `max_value` setting removes pairs where the minimum value change is above a specified value. -This is useful when an exchange has unbalanced limits. For example, if step-size = 1 (so you can only buy 1, or 2, or 3, but not 1.1 Coins) - and the price is pretty high (like 20$) as the coin has risen sharply since the last limit adaption. -As a result of the above, you can only buy for 20$, or 40$ - but not for 25$. +This is useful when an exchange has unbalanced limits. For example, if step-size = 1 (so you can only buy 1, or 2, or 3, but not 1.1 Coins) - and the price is pretty high (like 20\$) as the coin has risen sharply since the last limit adaption. +As a result of the above, you can only buy for 20\$, or 40\$ - but not for 25\$. On exchanges that deduct fees from the receiving currency (e.g. FTX) - this can result in high value coins / amounts that are unsellable as the amount is slightly below the limit. The `low_price_ratio` setting removes pairs where a raise of 1 price unit (pip) is above the `low_price_ratio` ratio. From 97a12ddab703ac98278b69b0cfac2ed7f90c0a9e Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Jun 2021 20:19:48 +0200 Subject: [PATCH 089/127] Version pin mkdocs to avoid nasty surprises fix use_directory_urls defaulting to false --- docs/requirements-docs.txt | 1 + mkdocs.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index ff07905b7..776d5592a 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,3 +1,4 @@ +mkdocs==1.2 mkdocs-material==7.1.7 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 diff --git a/mkdocs.yml b/mkdocs.yml index 2520ca929..cc5747225 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,7 @@ site_name: Freqtrade +site_url: https://www.freqtrade.io/ repo_url: https://github.com/freqtrade/freqtrade +use_directory_urls: True nav: - Home: index.md - Quickstart with Docker: docker_quickstart.md From 4512ece17db9708bdb238da9c1f29ca0c66dd1c8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 7 Jun 2021 21:05:25 +0200 Subject: [PATCH 090/127] Update Discord link --- .github/ISSUE_TEMPLATE/config.yml | 2 +- CONTRIBUTING.md | 2 +- README.md | 4 ++-- docs/developer.md | 2 +- docs/faq.md | 2 +- docs/index.md | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 7591c1fb0..5014de46a 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,5 +2,5 @@ blank_issues_enabled: false contact_links: - name: Discord Server - url: https://discord.gg/MA9v74M + url: https://discord.gg/p7nuUNVfP7 about: Ask a question or get community support from our Discord server diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c29d6e632..040cf3e98 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ Few pointers for contributions: - New features need to contain unit tests, must conform to PEP8 (max-line-length = 100) and should be documented with the introduction PR. - PR's can be declared as `[WIP]` - which signify Work in Progress Pull Requests (which are not finished). -If you are unsure, discuss the feature on our [discord server](https://discord.gg/MA9v74M), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. +If you are unsure, discuss the feature on our [discord server](https://discord.gg/p7nuUNVfP7), on [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) or in a [issue](https://github.com/freqtrade/freqtrade/issues) before a PR. ## Getting started diff --git a/README.md b/README.md index ab9597a77..d4b2df642 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ The project is currently setup in two main branches: For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. -Please check out our [discord server](https://discord.gg/MA9v74M). +Please check out our [discord server](https://discord.gg/p7nuUNVfP7). You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). @@ -178,7 +178,7 @@ to understand the requirements before sending your pull-requests. Coding is not a necessity to contribute - maybe start with improving our documentation? Issues labeled [good first issue](https://github.com/freqtrade/freqtrade/labels/good%20first%20issue) can be good first contributions, and will help get you familiar with the codebase. -**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/MA9v74M) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. +**Note** before starting any major new feature work, *please open an issue describing what you are planning to do* or talk to us on [discord](https://discord.gg/p7nuUNVfP7) or [Slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). This will ensure that interested parties can give valuable feedback on the feature, and let others know that you are working on it. **Important:** Always create your PR against the `develop` branch, not `stable`. diff --git a/docs/developer.md b/docs/developer.md index 4b8c64530..d0731a233 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -2,7 +2,7 @@ This page is intended for developers of Freqtrade, people who want to contribute to the Freqtrade codebase or documentation, or people who want to understand the source code of the application they're running. -All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/MA9v74M) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions. +All contributions, bug reports, bug fixes, documentation improvements, enhancements and ideas are welcome. We [track issues](https://github.com/freqtrade/freqtrade/issues) on [GitHub](https://github.com) and also have a dev channel on [discord](https://discord.gg/p7nuUNVfP7) or [slack](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) where you can ask questions. ## Documentation diff --git a/docs/faq.md b/docs/faq.md index 7233a92fe..e5da550fd 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -156,7 +156,7 @@ freqtrade hyperopt --hyperopt SampleHyperopt --hyperopt-loss SharpeHyperOptLossD ### Why does it take a long time to run hyperopt? -* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/MA9v74M). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. +* Discovering a great strategy with Hyperopt takes time. Study www.freqtrade.io, the Freqtrade Documentation page, join the Freqtrade [Slack community](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw) - or the Freqtrade [discord community](https://discord.gg/p7nuUNVfP7). While you patiently wait for the most advanced, free crypto bot in the world, to hand you a possible golden strategy specially designed just for you. * If you wonder why it can take from 20 minutes to days to do 1000 epochs here are some answers: diff --git a/docs/index.md b/docs/index.md index c2b6d5629..871172cfc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -76,7 +76,7 @@ Alternatively For any questions not covered by the documentation or for further information about the bot, or to simply engage with like-minded individuals, we encourage you to join our slack channel. -Please check out our [discord server](https://discord.gg/MA9v74M). +Please check out our [discord server](https://discord.gg/p7nuUNVfP7). You can also join our [Slack channel](https://join.slack.com/t/highfrequencybot/shared_invite/zt-mm786y93-Fxo37glxMY9g8OQC5AoOIw). From 35d6140068c2bbc3bf9ebe6e3b6bdf2bad1ddabf Mon Sep 17 00:00:00 2001 From: Bruno Gouvea Date: Mon, 7 Jun 2021 17:53:19 -0300 Subject: [PATCH 091/127] Displays the max drawdown in the hyper optimization results table. --- freqtrade/optimize/hyperopt_tools.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index ebe405d07..29014277e 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -233,18 +233,20 @@ class HyperoptTools(): 'results_metrics.winsdrawslosses', 'results_metrics.profit_mean', 'results_metrics.profit_total_abs', 'results_metrics.profit_total', 'results_metrics.holding_avg', - 'loss', 'is_initial_point', 'is_best']] + 'loss', 'is_initial_point', 'is_best', 'results_metrics.max_drawdown', + 'results_metrics.max_drawdown_abs']] else: # Legacy mode trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', 'results_metrics.winsdrawslosses', 'results_metrics.avg_profit', 'results_metrics.total_profit', 'results_metrics.profit', 'results_metrics.duration', - 'loss', 'is_initial_point', 'is_best']] + 'loss', 'is_initial_point', 'is_best', 'results_metrics.max_drawdown', + 'results_metrics.max_drawdown_abs']] trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', - 'Total profit', 'Profit', 'Avg duration', 'Objective', - 'is_initial_point', 'is_best'] + 'Total profit', 'Profit', 'Avg duration', 'Max Drawdown', 'max_drawdown_abs', + 'Objective', 'is_initial_point', 'is_best'] trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '* ' trials.loc[trials['is_best'], 'Best'] = 'Best' @@ -266,6 +268,16 @@ class HyperoptTools(): lambda x: f'{x:,.5f}'.rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ') ) + trials['Max Drawdown'] = trials.apply( + lambda x: '{} {}'.format( + round_coin_value(x['max_drawdown_abs'], stake_currency), + '({:,.2f}%)'.format(x['Max Drawdown'] * perc_multi).rjust(10, ' ') + ).rjust(25 + len(stake_currency)) + if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)), + axis=1 + ) + trials = trials.drop(columns=['max_drawdown_abs']) + stake_currency = config['stake_currency'] trials['Profit'] = trials.apply( lambda x: '{} {}'.format( From 5c3a418e6564d92c83cf123f150a6a87bb2cd81f Mon Sep 17 00:00:00 2001 From: Bruno Gouvea Date: Mon, 7 Jun 2021 18:15:26 -0300 Subject: [PATCH 092/127] Adjusting drawdown column position. --- freqtrade/optimize/hyperopt_tools.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 29014277e..9d6248643 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -233,16 +233,15 @@ class HyperoptTools(): 'results_metrics.winsdrawslosses', 'results_metrics.profit_mean', 'results_metrics.profit_total_abs', 'results_metrics.profit_total', 'results_metrics.holding_avg', - 'loss', 'is_initial_point', 'is_best', 'results_metrics.max_drawdown', - 'results_metrics.max_drawdown_abs']] + 'results_metrics.max_drawdown', 'results_metrics.max_drawdown_abs', + 'loss', 'is_initial_point', 'is_best']] else: # Legacy mode trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', 'results_metrics.winsdrawslosses', 'results_metrics.avg_profit', 'results_metrics.total_profit', - 'results_metrics.profit', 'results_metrics.duration', - 'loss', 'is_initial_point', 'is_best', 'results_metrics.max_drawdown', - 'results_metrics.max_drawdown_abs']] + 'results_metrics.profit', 'results_metrics.duration', 'results_metrics.max_drawdown', + 'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point', 'is_best']] trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', 'Total profit', 'Profit', 'Avg duration', 'Max Drawdown', 'max_drawdown_abs', @@ -268,6 +267,8 @@ class HyperoptTools(): lambda x: f'{x:,.5f}'.rjust(8, ' ') if x != 100000 else "N/A".rjust(8, ' ') ) + stake_currency = config['stake_currency'] + trials['Max Drawdown'] = trials.apply( lambda x: '{} {}'.format( round_coin_value(x['max_drawdown_abs'], stake_currency), @@ -278,7 +279,7 @@ class HyperoptTools(): ) trials = trials.drop(columns=['max_drawdown_abs']) - stake_currency = config['stake_currency'] + trials['Profit'] = trials.apply( lambda x: '{} {}'.format( round_coin_value(x['Total profit'], stake_currency), From c513c9685d4ff57f3a586430355fe221dc96c7af Mon Sep 17 00:00:00 2001 From: Bruno Gouvea Date: Mon, 7 Jun 2021 18:20:04 -0300 Subject: [PATCH 093/127] Remove blank line (PEP8) --- freqtrade/optimize/hyperopt_tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 9d6248643..59332ef69 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -279,7 +279,6 @@ class HyperoptTools(): ) trials = trials.drop(columns=['max_drawdown_abs']) - trials['Profit'] = trials.apply( lambda x: '{} {}'.format( round_coin_value(x['Total profit'], stake_currency), From 4595db39aa95b2da1d9d3c8d297072a6dfdf2fa2 Mon Sep 17 00:00:00 2001 From: Bruno Gouvea Date: Tue, 8 Jun 2021 02:18:00 -0300 Subject: [PATCH 094/127] Displaying max. drawdown only when it is not legacy mode. --- freqtrade/optimize/hyperopt_tools.py | 39 ++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 59332ef69..4243c182d 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -221,6 +221,7 @@ class HyperoptTools(): if 'results_metrics.winsdrawslosses' not in trials.columns: # Ensure compatibility with older versions of hyperopt results trials['results_metrics.winsdrawslosses'] = 'N/A' + legacy_mode = True if 'results_metrics.total_trades' in trials: @@ -235,17 +236,22 @@ class HyperoptTools(): 'results_metrics.profit_total', 'results_metrics.holding_avg', 'results_metrics.max_drawdown', 'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point', 'is_best']] + + trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', + 'Total profit', 'Profit', 'Avg duration', 'Max Drawdown', + 'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'] else: # Legacy mode trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', - 'results_metrics.winsdrawslosses', - 'results_metrics.avg_profit', 'results_metrics.total_profit', - 'results_metrics.profit', 'results_metrics.duration', 'results_metrics.max_drawdown', - 'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point', 'is_best']] + 'results_metrics.winsdrawslosses', 'results_metrics.avg_profit', + 'results_metrics.total_profit', 'results_metrics.profit', + 'results_metrics.duration', 'loss', 'is_initial_point', + 'is_best']] + + trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', + 'Total profit', 'Profit', 'Avg duration', 'Objective', + 'is_initial_point', 'is_best'] - trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', - 'Total profit', 'Profit', 'Avg duration', 'Max Drawdown', 'max_drawdown_abs', - 'Objective', 'is_initial_point', 'is_best'] trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '* ' trials.loc[trials['is_best'], 'Best'] = 'Best' @@ -269,15 +275,16 @@ class HyperoptTools(): stake_currency = config['stake_currency'] - trials['Max Drawdown'] = trials.apply( - lambda x: '{} {}'.format( - round_coin_value(x['max_drawdown_abs'], stake_currency), - '({:,.2f}%)'.format(x['Max Drawdown'] * perc_multi).rjust(10, ' ') - ).rjust(25 + len(stake_currency)) - if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)), - axis=1 - ) - trials = trials.drop(columns=['max_drawdown_abs']) + if not legacy_mode: + trials['Max Drawdown'] = trials.apply( + lambda x: '{} {}'.format( + round_coin_value(x['max_drawdown_abs'], stake_currency), + '({:,.2f}%)'.format(x['Max Drawdown'] * perc_multi).rjust(10, ' ') + ).rjust(25 + len(stake_currency)) + if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)), + axis=1 + ) + trials = trials.drop(columns=['max_drawdown_abs']) trials['Profit'] = trials.apply( lambda x: '{} {}'.format( From 816bb531b32e28dee9362395167bdbb05677a583 Mon Sep 17 00:00:00 2001 From: Bruno Gouvea Date: Tue, 8 Jun 2021 02:42:55 -0300 Subject: [PATCH 095/127] Creating fake column for legacy mode on max drawdown --- freqtrade/optimize/hyperopt_tools.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 4243c182d..a739ab9b9 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -222,6 +222,11 @@ class HyperoptTools(): # Ensure compatibility with older versions of hyperopt results trials['results_metrics.winsdrawslosses'] = 'N/A' + if 'results_metrics.max_drawdown_abs' not in trials.columns: + # Ensure compatibility with older versions of hyperopt results + trials['results_metrics.max_drawdown_abs'] = None + trials['results_metrics.max_drawdown'] = None + legacy_mode = True if 'results_metrics.total_trades' in trials: @@ -237,20 +242,18 @@ class HyperoptTools(): 'results_metrics.max_drawdown', 'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point', 'is_best']] - trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', - 'Total profit', 'Profit', 'Avg duration', 'Max Drawdown', - 'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'] else: # Legacy mode trials = trials[['Best', 'current_epoch', 'results_metrics.trade_count', 'results_metrics.winsdrawslosses', 'results_metrics.avg_profit', 'results_metrics.total_profit', 'results_metrics.profit', - 'results_metrics.duration', 'loss', 'is_initial_point', + 'results_metrics.duration', 'results_metrics.max_drawdown', + 'results_metrics.max_drawdown_abs', 'loss', 'is_initial_point', 'is_best']] - trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', - 'Total profit', 'Profit', 'Avg duration', 'Objective', - 'is_initial_point', 'is_best'] + trials.columns = ['Best', 'Epoch', 'Trades', ' Win Draw Loss', 'Avg profit', + 'Total profit', 'Profit', 'Avg duration', 'Max Drawdown', + 'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'] trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '* ' @@ -284,7 +287,10 @@ class HyperoptTools(): if x['Max Drawdown'] != 0.0 else '--'.rjust(25 + len(stake_currency)), axis=1 ) - trials = trials.drop(columns=['max_drawdown_abs']) + else: + trials = trials.drop(columns=['Max Drawdown']) + + trials = trials.drop(columns=['max_drawdown_abs']) trials['Profit'] = trials.apply( lambda x: '{} {}'.format( From 3cce668353322e3d134b94442d218141e22286b5 Mon Sep 17 00:00:00 2001 From: Bruno Gouvea Date: Tue, 8 Jun 2021 02:57:44 -0300 Subject: [PATCH 096/127] Creating a control variable to determine the existence of max drawdown in the final result. --- freqtrade/optimize/hyperopt_tools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index a739ab9b9..eaea4aa4a 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -222,10 +222,12 @@ class HyperoptTools(): # Ensure compatibility with older versions of hyperopt results trials['results_metrics.winsdrawslosses'] = 'N/A' + has_drawdown = True if 'results_metrics.max_drawdown_abs' not in trials.columns: # Ensure compatibility with older versions of hyperopt results trials['results_metrics.max_drawdown_abs'] = None trials['results_metrics.max_drawdown'] = None + has_drawdown = False legacy_mode = True @@ -278,7 +280,7 @@ class HyperoptTools(): stake_currency = config['stake_currency'] - if not legacy_mode: + if has_drawdown: trials['Max Drawdown'] = trials.apply( lambda x: '{} {}'.format( round_coin_value(x['max_drawdown_abs'], stake_currency), From 3310a4502977a730b34fac79c2d657d631edc2f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jun 2021 20:10:43 +0200 Subject: [PATCH 097/127] Change wording if limited lookback is used --- freqtrade/rpc/telegram.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 6c7fa0493..0a5125020 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -427,6 +427,7 @@ class Telegram(RPCHandler): fiat_disp_cur = self._config.get('fiat_display_currency', '') start_date = datetime.fromtimestamp(0) + timescale = None try: if context.args: timescale = int(context.args[0]) @@ -466,16 +467,18 @@ class Telegram(RPCHandler): else: markdown_msg = "`No closed trade` \n" - markdown_msg += (f"*ROI:* All trades\n" - f"∙ `{round_coin_value(profit_all_coin, stake_cur)} " - f"({profit_all_percent_mean:.2f}%) " - f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" - f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n" - f"*Total Trade Count:* `{trade_count}`\n" - f"*First Trade opened:* `{first_trade_date}`\n" - f"*Latest Trade opened:* `{latest_trade_date}\n`" - f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" - ) + markdown_msg += ( + f"*ROI:* All trades\n" + f"∙ `{round_coin_value(profit_all_coin, stake_cur)} " + f"({profit_all_percent_mean:.2f}%) " + f"({profit_all_percent_sum} \N{GREEK CAPITAL LETTER SIGMA}%)`\n" + f"∙ `{round_coin_value(profit_all_fiat, fiat_disp_cur)}`\n" + f"*Total Trade Count:* `{trade_count}`\n" + f"*{'First Trade opened' if not timescale else 'Showing Profit since'}:* " + f"`{first_trade_date}`\n" + f"*Latest Trade opened:* `{latest_trade_date}\n`" + f"*Win / Loss:* `{stats['winning_trades']} / {stats['losing_trades']}`" + ) if stats['closed_trade_count'] > 0: markdown_msg += (f"\n*Avg. Duration:* `{avg_duration}`\n" f"*Best Performing:* `{best_pair}: {best_rate:.2f}%`") From b9cf950bbffcfab687856b0ccde71d8cc9225654 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jun 2021 20:35:25 +0200 Subject: [PATCH 098/127] Add test for bad argument on /profit --- tests/rpc/test_rpc_telegram.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 7282c79bb..ae169e6e7 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -451,8 +451,10 @@ def test_profit_handle(default_conf, update, ticker, ticker_sell_up, fee, # Simulate fulfilled LIMIT_BUY order for trade trade.update(limit_buy_order) - - telegram._profit(update=update, context=MagicMock()) + context = MagicMock() + # Test with invalid 2nd argument (should silently pass) + context.args = ["aaa"] + telegram._profit(update=update, context=context) assert msg_mock.call_count == 1 assert 'No closed trade' in msg_mock.call_args_list[-1][0][0] assert '*ROI:* All trades' in msg_mock.call_args_list[-1][0][0] From d16a61948958ed1c2a05c777c3084e06b10b43ab Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jun 2021 21:04:34 +0200 Subject: [PATCH 099/127] Move SellType Enum to it's own module --- freqtrade/edge/edge_positioning.py | 3 ++- freqtrade/enums/__init__.py | 2 ++ freqtrade/enums/selltype.py | 21 +++++++++++++++++++ freqtrade/freqtradebot.py | 5 +++-- freqtrade/optimize/backtesting.py | 3 ++- .../plugins/protections/stoploss_guard.py | 2 +- freqtrade/rpc/rpc.py | 3 ++- freqtrade/strategy/interface.py | 20 +----------------- 8 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 freqtrade/enums/__init__.py create mode 100644 freqtrade/enums/selltype.py diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 4bc0d660b..418a12076 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -13,11 +13,12 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT from freqtrade.data.history import get_timerange, load_data, refresh_data +from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException from freqtrade.exchange.exchange import timeframe_to_seconds from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.state import RunMode -from freqtrade.strategy.interface import IStrategy, SellType +from freqtrade.strategy.interface import IStrategy logger = logging.getLogger(__name__) diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py new file mode 100644 index 000000000..f069c6152 --- /dev/null +++ b/freqtrade/enums/__init__.py @@ -0,0 +1,2 @@ +# flake8: noqa: F401 +from freqtrade.enums.selltype import SellType diff --git a/freqtrade/enums/selltype.py b/freqtrade/enums/selltype.py new file mode 100644 index 000000000..e9e3dcb5a --- /dev/null +++ b/freqtrade/enums/selltype.py @@ -0,0 +1,21 @@ + +from enum import Enum + + +class SellType(Enum): + """ + Enum to distinguish between sell reasons + """ + ROI = "roi" + STOP_LOSS = "stop_loss" + STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange" + TRAILING_STOP_LOSS = "trailing_stop_loss" + SELL_SIGNAL = "sell_signal" + FORCE_SELL = "force_sell" + EMERGENCY_SELL = "emergency_sell" + CUSTOM_SELL = "custom_sell" + NONE = "" + + def __str__(self): + # explicitly convert to String to help with exporting data. + return self.value diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 8628931b6..e854038d7 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,6 +16,7 @@ from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge +from freqtrade.enums import SellType from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds @@ -27,7 +28,7 @@ from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType from freqtrade.state import State -from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType +from freqtrade.strategy.interface import IStrategy, SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -337,7 +338,7 @@ class FreqtradeBot(LoggingMixin): # Assume this as the open order trade.open_order_id = order.order_id if fo: - logger.info(f"Found {order} for trade {trade}.jj") + logger.info(f"Found {order} for trade {trade}.") self.update_trade_state(trade, order.order_id, fo, stoploss_order=order.ft_order_side == 'stoploss') diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index cbc0995aa..922f89c22 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -17,6 +17,7 @@ from freqtrade.data import history from freqtrade.data.btanalysis import trade_list_to_dataframe from freqtrade.data.converter import trim_dataframes from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.mixins import LoggingMixin @@ -26,7 +27,7 @@ from freqtrade.persistence import LocalTrade, PairLocks, Trade from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.strategy.interface import IStrategy, SellCheckTuple, SellType +from freqtrade.strategy.interface import IStrategy, SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets diff --git a/freqtrade/plugins/protections/stoploss_guard.py b/freqtrade/plugins/protections/stoploss_guard.py index 635c0be04..45d393411 100644 --- a/freqtrade/plugins/protections/stoploss_guard.py +++ b/freqtrade/plugins/protections/stoploss_guard.py @@ -3,9 +3,9 @@ import logging from datetime import datetime, timedelta from typing import Any, Dict +from freqtrade.enums import SellType from freqtrade.persistence import Trade from freqtrade.plugins.protections import IProtection, ProtectionReturn -from freqtrade.strategy.interface import SellType logger = logging.getLogger(__name__) diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index be7d7e5ba..6245243aa 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -15,6 +15,7 @@ from pandas import DataFrame from freqtrade.configuration.timerange import TimeRange from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT from freqtrade.data.history import load_data +from freqtrade.enums import SellType from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.loggers import bufferHandler @@ -24,7 +25,7 @@ from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter from freqtrade.state import State -from freqtrade.strategy.interface import SellCheckTuple, SellType +from freqtrade.strategy.interface import SellCheckTuple logger = logging.getLogger(__name__) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index e2cde52eb..af1a57691 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -18,6 +18,7 @@ from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.persistence import PairLocks, Trade +from freqtrade.enums import SellType from freqtrade.strategy.hyper import HyperStrategyMixin from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets @@ -35,25 +36,6 @@ class SignalType(Enum): SELL = "sell" -class SellType(Enum): - """ - Enum to distinguish between sell reasons - """ - ROI = "roi" - STOP_LOSS = "stop_loss" - STOPLOSS_ON_EXCHANGE = "stoploss_on_exchange" - TRAILING_STOP_LOSS = "trailing_stop_loss" - SELL_SIGNAL = "sell_signal" - FORCE_SELL = "force_sell" - EMERGENCY_SELL = "emergency_sell" - CUSTOM_SELL = "custom_sell" - NONE = "" - - def __str__(self): - # explicitly convert to String to help with exporting data. - return self.value - - class SellCheckTuple(object): """ NamedTuple for Sell type + reason From 89b9915c12b9f7682e1961a7ba9487105a0ef4f7 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jun 2021 21:06:47 +0200 Subject: [PATCH 100/127] Update imports for SellType in tests --- freqtrade/strategy/interface.py | 2 +- tests/edge/test_edge.py | 2 +- tests/optimize/__init__.py | 2 +- tests/optimize/conftest.py | 2 +- tests/optimize/test_backtest_detail.py | 2 +- tests/optimize/test_backtesting.py | 2 +- tests/optimize/test_hyperopt.py | 2 +- tests/optimize/test_optimize_reports.py | 2 +- tests/plugins/test_protections.py | 2 +- tests/rpc/test_rpc_telegram.py | 2 +- tests/rpc/test_rpc_webhook.py | 2 +- tests/strategy/test_interface.py | 3 ++- tests/test_freqtradebot.py | 3 ++- tests/test_integration.py | 3 ++- 14 files changed, 17 insertions(+), 14 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index af1a57691..b60ec56f8 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -14,11 +14,11 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date from freqtrade.persistence import PairLocks, Trade -from freqtrade.enums import SellType from freqtrade.strategy.hyper import HyperStrategyMixin from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets diff --git a/tests/edge/test_edge.py b/tests/edge/test_edge.py index c4620e1c7..0655b3a0f 100644 --- a/tests/edge/test_edge.py +++ b/tests/edge/test_edge.py @@ -12,8 +12,8 @@ from pandas import DataFrame, to_datetime from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo +from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException -from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset) diff --git a/tests/optimize/__init__.py b/tests/optimize/__init__.py index 306850ff6..7cca2b1a8 100644 --- a/tests/optimize/__init__.py +++ b/tests/optimize/__init__.py @@ -3,8 +3,8 @@ from typing import Dict, List, NamedTuple, Optional import arrow from pandas import DataFrame +from freqtrade.enums import SellType from freqtrade.exchange import timeframe_to_minutes -from freqtrade.strategy.interface import SellType tests_start_time = arrow.get(2018, 10, 3) diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 11b4674f3..59ba3015d 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -5,9 +5,9 @@ from pathlib import Path import pandas as pd import pytest +from freqtrade.enums import SellType from freqtrade.optimize.hyperopt import Hyperopt from freqtrade.state import RunMode -from freqtrade.strategy.interface import SellType from tests.conftest import patch_exchange diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index c35a083ec..e5b969383 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -4,8 +4,8 @@ import logging import pytest from freqtrade.data.history import get_timerange +from freqtrade.enums import SellType from freqtrade.optimize.backtesting import Backtesting -from freqtrade.strategy.interface import SellType from tests.conftest import patch_exchange from tests.optimize import (BTContainer, BTrade, _build_backtest_dataframe, _get_frame_time_from_offset, tests_timeframe) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 632d458ce..ed8cf4bcf 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -16,12 +16,12 @@ from freqtrade.data.btanalysis import BT_DATA_COLUMNS, evaluate_result_multi from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange +from freqtrade.enums import SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade from freqtrade.resolvers import StrategyResolver from freqtrade.state import RunMode -from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index 07d208210..a1859cec3 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -13,6 +13,7 @@ from filelock import Timeout from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt from freqtrade.data.history import load_data +from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt import Hyperopt from freqtrade.optimize.hyperopt_auto import HyperOptAuto @@ -22,7 +23,6 @@ from freqtrade.optimize.space import SKDecimal from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver from freqtrade.state import RunMode from freqtrade.strategy.hyper import IntParameter -from freqtrade.strategy.interface import SellType from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) diff --git a/tests/optimize/test_optimize_reports.py b/tests/optimize/test_optimize_reports.py index f5c9a5a24..3f31efb95 100644 --- a/tests/optimize/test_optimize_reports.py +++ b/tests/optimize/test_optimize_reports.py @@ -12,6 +12,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, LAST_BT_RESULT_FN from freqtrade.data import history from freqtrade.data.btanalysis import get_latest_backtest_filename, load_backtest_data from freqtrade.edge import PairInfo +from freqtrade.enums import SellType from freqtrade.optimize.optimize_reports import (generate_backtest_stats, generate_daily_stats, generate_edge_table, generate_pair_metrics, generate_sell_reason_stats, @@ -20,7 +21,6 @@ from freqtrade.optimize.optimize_reports import (generate_backtest_stats, genera text_table_bt_results, text_table_sell_reason, text_table_strategy) from freqtrade.resolvers.strategy_resolver import StrategyResolver -from freqtrade.strategy.interface import SellType from tests.data.test_history import _backup_file, _clean_test_file diff --git a/tests/plugins/test_protections.py b/tests/plugins/test_protections.py index a39301145..10ab64690 100644 --- a/tests/plugins/test_protections.py +++ b/tests/plugins/test_protections.py @@ -4,9 +4,9 @@ from datetime import datetime, timedelta import pytest from freqtrade import constants +from freqtrade.enums import SellType from freqtrade.persistence import PairLocks, Trade from freqtrade.plugins.protectionmanager import ProtectionManager -from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has_re diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index ae169e6e7..ed952efad 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -17,6 +17,7 @@ from telegram.error import NetworkError from freqtrade import __version__ from freqtrade.constants import CANCEL_REASON from freqtrade.edge import PairInfo +from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.loggers import setup_logging @@ -24,7 +25,6 @@ from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC, RPCMessageType from freqtrade.rpc.telegram import Telegram, authorized_only from freqtrade.state import RunMode, State -from freqtrade.strategy.interface import SellType from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, patch_exchange, patch_get_signal, patch_whitelist) diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 0560f8d53..43d5dbdc5 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -5,9 +5,9 @@ from unittest.mock import MagicMock import pytest from requests import RequestException +from freqtrade.enums import SellType from freqtrade.rpc import RPC, RPCMessageType from freqtrade.rpc.webhook import Webhook -from freqtrade.strategy.interface import SellType from tests.conftest import get_patched_freqtradebot, log_has diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index df487986f..64081fa37 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -10,12 +10,13 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data +from freqtrade.enums import SellType from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.persistence import PairLocks, Trade from freqtrade.resolvers import StrategyResolver from freqtrade.strategy.hyper import (BaseParameter, CategoricalParameter, DecimalParameter, IntParameter, RealParameter) -from freqtrade.strategy.interface import SellCheckTuple, SellType +from freqtrade.strategy.interface import SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from tests.conftest import log_has, log_has_re diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 66866a8fc..6d74c3844 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -11,6 +11,7 @@ import arrow import pytest from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT +from freqtrade.enums import SellType from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, TemporaryError) @@ -19,7 +20,7 @@ from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.rpc import RPCMessageType from freqtrade.state import RunMode, State -from freqtrade.strategy.interface import SellCheckTuple, SellType +from freqtrade.strategy.interface import SellCheckTuple from freqtrade.worker import Worker from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, log_has, log_has_re, patch_edge, patch_exchange, patch_get_signal, diff --git a/tests/test_integration.py b/tests/test_integration.py index 33b3e1140..b12959a03 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,9 +2,10 @@ from unittest.mock import MagicMock import pytest +from freqtrade.enums import SellType from freqtrade.persistence import Trade from freqtrade.rpc.rpc import RPC -from freqtrade.strategy.interface import SellCheckTuple, SellType +from freqtrade.strategy.interface import SellCheckTuple from tests.conftest import get_patched_freqtradebot, patch_get_signal From 3c149b9b5921d4987da97a9934515ae008cf844b Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jun 2021 21:09:39 +0200 Subject: [PATCH 101/127] Move signalType to enums --- freqtrade/enums/__init__.py | 1 + freqtrade/enums/signaltype.py | 9 +++++++++ freqtrade/strategy/interface.py | 10 +--------- 3 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 freqtrade/enums/signaltype.py diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py index f069c6152..b4037ec12 100644 --- a/freqtrade/enums/__init__.py +++ b/freqtrade/enums/__init__.py @@ -1,2 +1,3 @@ # flake8: noqa: F401 from freqtrade.enums.selltype import SellType +from freqtrade.enums.signaltype import SignalType diff --git a/freqtrade/enums/signaltype.py b/freqtrade/enums/signaltype.py new file mode 100644 index 000000000..d636f378a --- /dev/null +++ b/freqtrade/enums/signaltype.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class SignalType(Enum): + """ + Enum to distinguish between buy and sell signals + """ + BUY = "buy" + SELL = "sell" diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index b60ec56f8..834495a19 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -14,7 +14,7 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums import SellType +from freqtrade.enums import SellType, SignalType from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date @@ -28,14 +28,6 @@ logger = logging.getLogger(__name__) CUSTOM_SELL_MAX_LENGTH = 64 -class SignalType(Enum): - """ - Enum to distinguish between buy and sell signals - """ - BUY = "buy" - SELL = "sell" - - class SellCheckTuple(object): """ NamedTuple for Sell type + reason From 9c34304cb979de092885e637e83a675784b2bdd8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jun 2021 21:20:35 +0200 Subject: [PATCH 102/127] Move state enums to enums package --- freqtrade/commands/data_commands.py | 2 +- freqtrade/commands/deploy_commands.py | 2 +- freqtrade/commands/hyperopt_commands.py | 2 +- freqtrade/commands/list_commands.py | 2 +- freqtrade/commands/optimize_commands.py | 2 +- freqtrade/commands/pairlist_commands.py | 2 +- freqtrade/commands/plot_commands.py | 2 +- freqtrade/configuration/check_exchange.py | 2 +- freqtrade/configuration/config_setup.py | 2 +- freqtrade/configuration/config_validation.py | 2 +- freqtrade/configuration/configuration.py | 2 +- freqtrade/data/dataprovider.py | 2 +- freqtrade/edge/edge_positioning.py | 3 +-- freqtrade/enums/__init__.py | 2 ++ freqtrade/{state.py => enums/runmode.py} | 12 ------------ freqtrade/enums/state.py | 18 ++++++++++++++++++ freqtrade/freqtradebot.py | 3 +-- freqtrade/rpc/rpc.py | 3 +-- freqtrade/strategy/hyper.py | 2 +- freqtrade/wallets.py | 2 +- freqtrade/worker.py | 2 +- tests/commands/test_commands.py | 2 +- tests/conftest.py | 2 +- tests/data/test_dataprovider.py | 2 +- tests/optimize/conftest.py | 3 +-- tests/optimize/test_backtesting.py | 3 +-- tests/optimize/test_edge_cli.py | 2 +- tests/optimize/test_hyperopt.py | 3 +-- tests/rpc/test_rpc.py | 2 +- tests/rpc/test_rpc_apiserver.py | 2 +- tests/rpc/test_rpc_telegram.py | 3 +-- tests/test_configuration.py | 2 +- tests/test_freqtradebot.py | 3 +-- tests/test_main.py | 2 +- tests/test_worker.py | 2 +- 35 files changed, 52 insertions(+), 52 deletions(-) rename freqtrade/{state.py => enums/runmode.py} (78%) create mode 100644 freqtrade/enums/state.py diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 58191ddb4..3877e0801 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -8,11 +8,11 @@ from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/deploy_commands.py b/freqtrade/commands/deploy_commands.py index 5ba3db9f9..cc0d653b9 100644 --- a/freqtrade/commands/deploy_commands.py +++ b/freqtrade/commands/deploy_commands.py @@ -8,9 +8,9 @@ import requests from freqtrade.configuration import setup_utils_configuration from freqtrade.configuration.directory_operations import copy_sample_files, create_userdata_dir from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import render_template, render_template_with_fallback -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index d8b00f369..dba4179ff 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -6,9 +6,9 @@ from colorama import init as colorama_init from freqtrade.configuration import setup_utils_configuration from freqtrade.data.btanalysis import get_latest_hyperopt_file +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.optimize.optimize_reports import show_backtest_result -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index 167847a0d..f59adb5d8 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -12,11 +12,11 @@ from tabulate import tabulate from freqtrade.configuration import setup_utils_configuration from freqtrade.constants import USERPATH_HYPEROPTS, USERPATH_STRATEGIES +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import market_is_active, validate_exchanges from freqtrade.misc import plural from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/optimize_commands.py b/freqtrade/commands/optimize_commands.py index 6323bc2b1..a84b3b3bd 100644 --- a/freqtrade/commands/optimize_commands.py +++ b/freqtrade/commands/optimize_commands.py @@ -3,9 +3,9 @@ from typing import Any, Dict from freqtrade import constants from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.misc import round_coin_value -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/pairlist_commands.py b/freqtrade/commands/pairlist_commands.py index a2b10a42f..9f7a5958e 100644 --- a/freqtrade/commands/pairlist_commands.py +++ b/freqtrade/commands/pairlist_commands.py @@ -4,8 +4,8 @@ from typing import Any, Dict import rapidjson from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.resolvers import ExchangeResolver -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/commands/plot_commands.py b/freqtrade/commands/plot_commands.py index 5e547acb0..73a429a28 100644 --- a/freqtrade/commands/plot_commands.py +++ b/freqtrade/commands/plot_commands.py @@ -1,8 +1,8 @@ from typing import Any, Dict from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode def validate_plot_args(args: Dict[str, Any]) -> None: diff --git a/freqtrade/configuration/check_exchange.py b/freqtrade/configuration/check_exchange.py index 832caf153..f282447d4 100644 --- a/freqtrade/configuration/check_exchange.py +++ b/freqtrade/configuration/check_exchange.py @@ -1,10 +1,10 @@ import logging from typing import Any, Dict +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import (available_exchanges, is_exchange_known_ccxt, is_exchange_officially_supported, validate_exchange) -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/config_setup.py b/freqtrade/configuration/config_setup.py index 3b0f778f4..cd8464ead 100644 --- a/freqtrade/configuration/config_setup.py +++ b/freqtrade/configuration/config_setup.py @@ -1,7 +1,7 @@ import logging from typing import Any, Dict -from freqtrade.state import RunMode +from freqtrade.enums import RunMode from .check_exchange import remove_credentials from .config_validation import validate_config_consistency diff --git a/freqtrade/configuration/config_validation.py b/freqtrade/configuration/config_validation.py index 31e38d572..3004d6bf7 100644 --- a/freqtrade/configuration/config_validation.py +++ b/freqtrade/configuration/config_validation.py @@ -6,8 +6,8 @@ from jsonschema import Draft4Validator, validators from jsonschema.exceptions import ValidationError, best_match from freqtrade import constants +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index abb9f994b..bfeb2da5c 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -12,10 +12,10 @@ from freqtrade.configuration.check_exchange import check_exchange from freqtrade.configuration.deprecated_settings import process_temporary_deprecated_settings from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir from freqtrade.configuration.load_config import load_config_file, load_file +from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts -from freqtrade.state import NON_UTIL_MODES, TRADING_MODES, RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 1a86eece5..391ed4587 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -12,9 +12,9 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe from freqtrade.data.history import load_pair_history +from freqtrade.enums import RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.exchange import Exchange -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index 418a12076..9466a1649 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -13,11 +13,10 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import DATETIME_PRINT_FORMAT, UNLIMITED_STAKE_AMOUNT from freqtrade.data.history import get_timerange, load_data, refresh_data -from freqtrade.enums import SellType +from freqtrade.enums import RunMode, SellType from freqtrade.exceptions import OperationalException from freqtrade.exchange.exchange import timeframe_to_seconds from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist -from freqtrade.state import RunMode from freqtrade.strategy.interface import IStrategy diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py index b4037ec12..f0092ffdf 100644 --- a/freqtrade/enums/__init__.py +++ b/freqtrade/enums/__init__.py @@ -1,3 +1,5 @@ # flake8: noqa: F401 +from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode from freqtrade.enums.selltype import SellType from freqtrade.enums.signaltype import SignalType +from freqtrade.enums.state import State diff --git a/freqtrade/state.py b/freqtrade/enums/runmode.py similarity index 78% rename from freqtrade/state.py rename to freqtrade/enums/runmode.py index 8ddff71d9..06ebf91a2 100644 --- a/freqtrade/state.py +++ b/freqtrade/enums/runmode.py @@ -6,18 +6,6 @@ Bot state constant from enum import Enum -class State(Enum): - """ - Bot application states - """ - RUNNING = 1 - STOPPED = 2 - RELOAD_CONFIG = 3 - - def __str__(self): - return f"{self.name.lower()}" - - class RunMode(Enum): """ Bot running mode (backtest, hyperopt, ...) diff --git a/freqtrade/enums/state.py b/freqtrade/enums/state.py new file mode 100644 index 000000000..d6c611ea0 --- /dev/null +++ b/freqtrade/enums/state.py @@ -0,0 +1,18 @@ +# pragma pylint: disable=too-few-public-methods + +""" +Bot state constant +""" +from enum import Enum + + +class State(Enum): + """ + Bot application states + """ + RUNNING = 1 + STOPPED = 2 + RELOAD_CONFIG = 3 + + def __str__(self): + return f"{self.name.lower()}" diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index e854038d7..1efe277cc 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,7 +16,7 @@ from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.enums import SellType +from freqtrade.enums import SellType, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds @@ -27,7 +27,6 @@ from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.rpc import RPCManager, RPCMessageType -from freqtrade.state import State from freqtrade.strategy.interface import IStrategy, SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 6245243aa..0b7a4152a 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -15,7 +15,7 @@ from pandas import DataFrame from freqtrade.configuration.timerange import TimeRange from freqtrade.constants import CANCEL_REASON, DATETIME_PRINT_FORMAT from freqtrade.data.history import load_data -from freqtrade.enums import SellType +from freqtrade.enums import SellType, State from freqtrade.exceptions import ExchangeError, PricingError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_msecs from freqtrade.loggers import bufferHandler @@ -24,7 +24,6 @@ from freqtrade.persistence import PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.rpc.fiat_convert import CryptoToFiatConverter -from freqtrade.state import State from freqtrade.strategy.interface import SellCheckTuple diff --git a/freqtrade/strategy/hyper.py b/freqtrade/strategy/hyper.py index a320f2bb8..e487ffeff 100644 --- a/freqtrade/strategy/hyper.py +++ b/freqtrade/strategy/hyper.py @@ -14,8 +14,8 @@ with suppress(ImportError): from skopt.space import Integer, Real, Categorical from freqtrade.optimize.space import SKDecimal +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/wallets.py b/freqtrade/wallets.py index d5f979c24..1b2ec4550 100644 --- a/freqtrade/wallets.py +++ b/freqtrade/wallets.py @@ -8,10 +8,10 @@ from typing import Any, Dict, NamedTuple import arrow from freqtrade.constants import UNLIMITED_STAKE_AMOUNT +from freqtrade.enums import RunMode from freqtrade.exceptions import DependencyException from freqtrade.exchange import Exchange from freqtrade.persistence import LocalTrade, Trade -from freqtrade.state import RunMode logger = logging.getLogger(__name__) diff --git a/freqtrade/worker.py b/freqtrade/worker.py index ec9331eef..4fa9166bf 100755 --- a/freqtrade/worker.py +++ b/freqtrade/worker.py @@ -11,9 +11,9 @@ import sdnotify from freqtrade import __version__, constants from freqtrade.configuration import Configuration +from freqtrade.enums import State from freqtrade.exceptions import OperationalException, TemporaryError from freqtrade.freqtradebot import FreqtradeBot -from freqtrade.state import State logger = logging.getLogger(__name__) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index b0808ae06..47f298ad7 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -17,8 +17,8 @@ from freqtrade.commands import (start_convert_data, start_create_userdir, start_ from freqtrade.commands.deploy_commands import (clean_ui_subdir, download_and_install_ui, get_ui_download_url, read_ui_version) from freqtrade.configuration import setup_utils_configuration +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException -from freqtrade.state import RunMode from tests.conftest import (create_mock_trades, get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) from tests.conftest_trades import MOCK_TRADE_COUNT diff --git a/tests/conftest.py b/tests/conftest.py index 43a98647f..dd38ca610 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,11 +17,11 @@ from freqtrade import constants from freqtrade.commands import Arguments from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import Edge, PairInfo +from freqtrade.enums import RunMode from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import LocalTrade, Trade, init_db from freqtrade.resolvers import ExchangeResolver -from freqtrade.state import RunMode from freqtrade.worker import Worker from tests.conftest_trades import (mock_trade_1, mock_trade_2, mock_trade_3, mock_trade_4, mock_trade_5, mock_trade_6) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index b87258c64..e43309743 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -5,9 +5,9 @@ import pytest from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums import RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.plugins.pairlistmanager import PairListManager -from freqtrade.state import RunMode from tests.conftest import get_patched_exchange diff --git a/tests/optimize/conftest.py b/tests/optimize/conftest.py index 59ba3015d..a7fd238d1 100644 --- a/tests/optimize/conftest.py +++ b/tests/optimize/conftest.py @@ -5,9 +5,8 @@ from pathlib import Path import pandas as pd import pytest -from freqtrade.enums import SellType +from freqtrade.enums import RunMode, SellType from freqtrade.optimize.hyperopt import Hyperopt -from freqtrade.state import RunMode from tests.conftest import patch_exchange diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index ed8cf4bcf..7387c8865 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -16,12 +16,11 @@ from freqtrade.data.btanalysis import BT_DATA_COLUMNS, evaluate_result_multi from freqtrade.data.converter import clean_ohlcv_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import get_timerange -from freqtrade.enums import SellType +from freqtrade.enums import RunMode, SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.optimize.backtesting import Backtesting from freqtrade.persistence import LocalTrade from freqtrade.resolvers import StrategyResolver -from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) diff --git a/tests/optimize/test_edge_cli.py b/tests/optimize/test_edge_cli.py index 188b4aa5f..6818a573b 100644 --- a/tests/optimize/test_edge_cli.py +++ b/tests/optimize/test_edge_cli.py @@ -4,8 +4,8 @@ from unittest.mock import MagicMock from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_edge +from freqtrade.enums import RunMode from freqtrade.optimize.edge_cli import EdgeCli -from freqtrade.state import RunMode from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index a1859cec3..e5c05e4c0 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -13,7 +13,7 @@ from filelock import Timeout from freqtrade.commands.optimize_commands import setup_optimize_configuration, start_hyperopt from freqtrade.data.history import load_data -from freqtrade.enums import SellType +from freqtrade.enums import RunMode, SellType from freqtrade.exceptions import OperationalException from freqtrade.optimize.hyperopt import Hyperopt from freqtrade.optimize.hyperopt_auto import HyperOptAuto @@ -21,7 +21,6 @@ from freqtrade.optimize.hyperopt_tools import HyperoptTools from freqtrade.optimize.optimize_reports import generate_strategy_stats from freqtrade.optimize.space import SKDecimal from freqtrade.resolvers.hyperopt_resolver import HyperOptResolver -from freqtrade.state import RunMode from freqtrade.strategy.hyper import IntParameter from tests.conftest import (get_args, log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) diff --git a/tests/rpc/test_rpc.py b/tests/rpc/test_rpc.py index 7556dde6d..8a41099f4 100644 --- a/tests/rpc/test_rpc.py +++ b/tests/rpc/test_rpc.py @@ -8,12 +8,12 @@ import pytest from numpy import isnan from freqtrade.edge import PairInfo +from freqtrade.enums import State from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError from freqtrade.persistence import Trade from freqtrade.persistence.pairlock_middleware import PairLocks from freqtrade.rpc import RPC, RPCException from freqtrade.rpc.fiat_convert import CryptoToFiatConverter -from freqtrade.state import State from tests.conftest import create_mock_trades, get_patched_freqtradebot, patch_get_signal diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index def2e43c6..8d48e2286 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -15,6 +15,7 @@ from numpy import isnan from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ +from freqtrade.enums import RunMode, State from freqtrade.exceptions import ExchangeError from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade @@ -22,7 +23,6 @@ from freqtrade.rpc import RPC from freqtrade.rpc.api_server import ApiServer from freqtrade.rpc.api_server.api_auth import create_token, get_user_from_token from freqtrade.rpc.api_server.uvicorn_threaded import UvicornServer -from freqtrade.state import RunMode, State from tests.conftest import (create_mock_trades, get_mock_coro, get_patched_freqtradebot, log_has, log_has_re, patch_get_signal) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index ed952efad..7ff9cda9e 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -17,14 +17,13 @@ from telegram.error import NetworkError from freqtrade import __version__ from freqtrade.constants import CANCEL_REASON from freqtrade.edge import PairInfo -from freqtrade.enums import SellType +from freqtrade.enums import RunMode, SellType, State from freqtrade.exceptions import OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.loggers import setup_logging from freqtrade.persistence import PairLocks, Trade from freqtrade.rpc import RPC, RPCMessageType from freqtrade.rpc.telegram import Telegram, authorized_only -from freqtrade.state import RunMode, State from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, patch_exchange, patch_get_signal, patch_whitelist) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index b08d0775c..aa121edfa 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -20,9 +20,9 @@ from freqtrade.configuration.deprecated_settings import (check_conflicting_setti process_temporary_deprecated_settings) from freqtrade.configuration.load_config import load_config_file, load_file, log_config_error_range from freqtrade.constants import DEFAULT_DB_DRYRUN_URL, DEFAULT_DB_PROD_URL +from freqtrade.enums import RunMode from freqtrade.exceptions import OperationalException from freqtrade.loggers import _set_loggers, setup_logging, setup_logging_pre -from freqtrade.state import RunMode from tests.conftest import log_has, log_has_re, patched_configuration_load_config_file diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 6d74c3844..1fecb3f6a 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -11,7 +11,7 @@ import arrow import pytest from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT -from freqtrade.enums import SellType +from freqtrade.enums import RunMode, SellType, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, TemporaryError) @@ -19,7 +19,6 @@ from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock from freqtrade.rpc import RPCMessageType -from freqtrade.state import RunMode, State from freqtrade.strategy.interface import SellCheckTuple from freqtrade.worker import Worker from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, diff --git a/tests/test_main.py b/tests/test_main.py index d52dcaf79..3546a3bab 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,10 +7,10 @@ from unittest.mock import MagicMock, PropertyMock import pytest from freqtrade.commands import Arguments +from freqtrade.enums import State from freqtrade.exceptions import FreqtradeException, OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.main import main -from freqtrade.state import State from freqtrade.worker import Worker from tests.conftest import (log_has, log_has_re, patch_exchange, patched_configuration_load_config_file) diff --git a/tests/test_worker.py b/tests/test_worker.py index 839f7cdac..c3773d296 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -3,7 +3,7 @@ import time from unittest.mock import MagicMock, PropertyMock from freqtrade.data.dataprovider import DataProvider -from freqtrade.state import State +from freqtrade.enums import State from freqtrade.worker import Worker from tests.conftest import get_patched_worker, log_has, log_has_re From 756904f9854abfd43e71d7df7677f217ab1f07a3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 8 Jun 2021 21:21:29 +0200 Subject: [PATCH 103/127] Set sell_reason to stoploss when closing the trade as stoploss closes #5101 --- freqtrade/persistence/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index ee934f657..984b2de27 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -14,6 +14,7 @@ from sqlalchemy.pool import StaticPool from sqlalchemy.sql.schema import UniqueConstraint from freqtrade.constants import DATETIME_PRINT_FORMAT +from freqtrade.enums import SellType from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.misc import safe_value_fallback from freqtrade.persistence.migrations import check_migrate @@ -430,6 +431,7 @@ class LocalTrade(): elif order_type in ('stop_loss_limit', 'stop-loss', 'stop-loss-limit', 'stop'): self.stoploss_order_id = None self.close_rate_requested = self.stop_loss + self.sell_reason = SellType.STOPLOSS_ON_EXCHANGE.value if self.is_open: logger.info(f'{order_type.upper()} is hit for {self}.') self.close(safe_value_fallback(order, 'average', 'price')) From 40f1ede77549ee7a57645030812507a326378fb8 Mon Sep 17 00:00:00 2001 From: Bruno Gouvea Date: Wed, 9 Jun 2021 12:03:24 -0300 Subject: [PATCH 104/127] Simplifying HO's result function --- freqtrade/optimize/hyperopt_tools.py | 40 ++++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index eaea4aa4a..6024422bc 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -206,33 +206,20 @@ class HyperoptTools(): f"Objective: {results['loss']:.5f}") @staticmethod - def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, - print_colorized: bool, remove_header: int) -> str: - """ - Log result table - """ - if not results: - return '' + def prepare_trials_columns(trials, legacy_mode: bool, has_drawdown: bool) -> str: - tabulate.PRESERVE_WHITESPACE = True - - trials = json_normalize(results, max_level=1) trials['Best'] = '' + if 'results_metrics.winsdrawslosses' not in trials.columns: # Ensure compatibility with older versions of hyperopt results trials['results_metrics.winsdrawslosses'] = 'N/A' - has_drawdown = True - if 'results_metrics.max_drawdown_abs' not in trials.columns: + if not has_drawdown: # Ensure compatibility with older versions of hyperopt results trials['results_metrics.max_drawdown_abs'] = None trials['results_metrics.max_drawdown'] = None - has_drawdown = False - legacy_mode = True - - if 'results_metrics.total_trades' in trials: - legacy_mode = False + if not legacy_mode: # New mode, using backtest result for metrics trials['results_metrics.winsdrawslosses'] = trials.apply( lambda x: f"{x['results_metrics.wins']} {x['results_metrics.draws']:>4} " @@ -257,6 +244,25 @@ class HyperoptTools(): 'Total profit', 'Profit', 'Avg duration', 'Max Drawdown', 'max_drawdown_abs', 'Objective', 'is_initial_point', 'is_best'] + return trials + + @staticmethod + def get_result_table(config: dict, results: list, total_epochs: int, highlight_best: bool, + print_colorized: bool, remove_header: int) -> str: + """ + Log result table + """ + if not results: + return '' + + tabulate.PRESERVE_WHITESPACE = True + trials = json_normalize(results, max_level=1) + + legacy_mode = 'results_metrics.total_trades' not in trials + has_drawdown = 'results_metrics.max_drawdown_abs' in trials.columns + + trials = HyperoptTools.prepare_trials_columns(trials, legacy_mode, has_drawdown) + trials['is_profit'] = False trials.loc[trials['is_initial_point'], 'Best'] = '* ' trials.loc[trials['is_best'], 'Best'] = 'Best' From d4dfdf04fc1ba98710192ee9a83086e6e7bbeaba Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 9 Jun 2021 19:51:44 +0200 Subject: [PATCH 105/127] Move RPCMessageType to enums --- freqtrade/enums/__init__.py | 1 + freqtrade/enums/rpcmessagetype.py | 19 +++++++++++++++++++ freqtrade/enums/runmode.py | 5 ----- freqtrade/freqtradebot.py | 4 ++-- freqtrade/rpc/__init__.py | 2 +- freqtrade/rpc/rpc.py | 19 ------------------- freqtrade/rpc/rpc_manager.py | 3 ++- freqtrade/rpc/telegram.py | 3 ++- freqtrade/rpc/webhook.py | 3 ++- freqtrade/strategy/interface.py | 1 - tests/rpc/test_rpc_manager.py | 3 ++- tests/rpc/test_rpc_telegram.py | 4 ++-- tests/rpc/test_rpc_webhook.py | 4 ++-- tests/test_freqtradebot.py | 3 +-- 14 files changed, 36 insertions(+), 38 deletions(-) create mode 100644 freqtrade/enums/rpcmessagetype.py diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py index f0092ffdf..78163d86f 100644 --- a/freqtrade/enums/__init__.py +++ b/freqtrade/enums/__init__.py @@ -1,4 +1,5 @@ # flake8: noqa: F401 +from freqtrade.enums.rpcmessagetype import RPCMessageType from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode from freqtrade.enums.selltype import SellType from freqtrade.enums.signaltype import SignalType diff --git a/freqtrade/enums/rpcmessagetype.py b/freqtrade/enums/rpcmessagetype.py new file mode 100644 index 000000000..9c59f6108 --- /dev/null +++ b/freqtrade/enums/rpcmessagetype.py @@ -0,0 +1,19 @@ +from enum import Enum + + +class RPCMessageType(Enum): + STATUS = 'status' + WARNING = 'warning' + STARTUP = 'startup' + BUY = 'buy' + BUY_FILL = 'buy_fill' + BUY_CANCEL = 'buy_cancel' + SELL = 'sell' + SELL_FILL = 'sell_fill' + SELL_CANCEL = 'sell_cancel' + + def __repr__(self): + return self.value + + def __str__(self): + return self.value diff --git a/freqtrade/enums/runmode.py b/freqtrade/enums/runmode.py index 06ebf91a2..7826d1d0c 100644 --- a/freqtrade/enums/runmode.py +++ b/freqtrade/enums/runmode.py @@ -1,8 +1,3 @@ -# pragma pylint: disable=too-few-public-methods - -""" -Bot state constant -""" from enum import Enum diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 1efe277cc..7aca0fcb2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -16,7 +16,7 @@ from freqtrade.configuration import validate_config_consistency from freqtrade.data.converter import order_book_to_dataframe from freqtrade.data.dataprovider import DataProvider from freqtrade.edge import Edge -from freqtrade.enums import SellType, State +from freqtrade.enums import RPCMessageType, SellType, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, PricingError) from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds @@ -26,7 +26,7 @@ from freqtrade.persistence import Order, PairLocks, Trade, cleanup_db, init_db from freqtrade.plugins.pairlistmanager import PairListManager from freqtrade.plugins.protectionmanager import ProtectionManager from freqtrade.resolvers import ExchangeResolver, StrategyResolver -from freqtrade.rpc import RPCManager, RPCMessageType +from freqtrade.rpc import RPCManager from freqtrade.strategy.interface import IStrategy, SellCheckTuple from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets diff --git a/freqtrade/rpc/__init__.py b/freqtrade/rpc/__init__.py index 0a0130ca7..957565e2c 100644 --- a/freqtrade/rpc/__init__.py +++ b/freqtrade/rpc/__init__.py @@ -1,3 +1,3 @@ # flake8: noqa: F401 -from .rpc import RPC, RPCException, RPCHandler, RPCMessageType +from .rpc import RPC, RPCException, RPCHandler from .rpc_manager import RPCManager diff --git a/freqtrade/rpc/rpc.py b/freqtrade/rpc/rpc.py index 0b7a4152a..2a7721af0 100644 --- a/freqtrade/rpc/rpc.py +++ b/freqtrade/rpc/rpc.py @@ -4,7 +4,6 @@ This module contains class to define a RPC communications import logging from abc import abstractmethod from datetime import date, datetime, timedelta, timezone -from enum import Enum from math import isnan from typing import Any, Dict, List, Optional, Tuple, Union @@ -30,24 +29,6 @@ from freqtrade.strategy.interface import SellCheckTuple logger = logging.getLogger(__name__) -class RPCMessageType(Enum): - STATUS = 'status' - WARNING = 'warning' - STARTUP = 'startup' - BUY = 'buy' - BUY_FILL = 'buy_fill' - BUY_CANCEL = 'buy_cancel' - SELL = 'sell' - SELL_FILL = 'sell_fill' - SELL_CANCEL = 'sell_cancel' - - def __repr__(self): - return self.value - - def __str__(self): - return self.value - - class RPCException(Exception): """ Should be raised with a rpc-formatted message in an _rpc_* method diff --git a/freqtrade/rpc/rpc_manager.py b/freqtrade/rpc/rpc_manager.py index f819b55b4..18ed68041 100644 --- a/freqtrade/rpc/rpc_manager.py +++ b/freqtrade/rpc/rpc_manager.py @@ -4,7 +4,8 @@ This module contains class to manage RPC communications (Telegram, Slack, ...) import logging from typing import Any, Dict, List -from freqtrade.rpc import RPC, RPCHandler, RPCMessageType +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPC, RPCHandler logger = logging.getLogger(__name__) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 8742d41e2..c3ddfd644 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -22,9 +22,10 @@ from telegram.utils.helpers import escape_markdown from freqtrade.__init__ import __version__ from freqtrade.constants import DUST_PER_COIN +from freqtrade.enums import RPCMessageType from freqtrade.exceptions import OperationalException from freqtrade.misc import chunks, round_coin_value -from freqtrade.rpc import RPC, RPCException, RPCHandler, RPCMessageType +from freqtrade.rpc import RPC, RPCException, RPCHandler logger = logging.getLogger(__name__) diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 24e1348f1..0e4a4bf6f 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -6,7 +6,8 @@ from typing import Any, Dict from requests import RequestException, post -from freqtrade.rpc import RPC, RPCHandler, RPCMessageType +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPC, RPCHandler logger = logging.getLogger(__name__) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 834495a19..8ea38f503 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -6,7 +6,6 @@ import logging import warnings from abc import ABC, abstractmethod from datetime import datetime, timedelta, timezone -from enum import Enum from typing import Dict, List, Optional, Tuple, Union import arrow diff --git a/tests/rpc/test_rpc_manager.py b/tests/rpc/test_rpc_manager.py index 69a757fcf..918022386 100644 --- a/tests/rpc/test_rpc_manager.py +++ b/tests/rpc/test_rpc_manager.py @@ -3,7 +3,8 @@ import logging import time from unittest.mock import MagicMock -from freqtrade.rpc import RPCManager, RPCMessageType +from freqtrade.enums import RPCMessageType +from freqtrade.rpc import RPCManager from tests.conftest import get_patched_freqtradebot, log_has diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 7ff9cda9e..bbda55c3e 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -17,12 +17,12 @@ from telegram.error import NetworkError from freqtrade import __version__ from freqtrade.constants import CANCEL_REASON from freqtrade.edge import PairInfo -from freqtrade.enums import RunMode, SellType, State +from freqtrade.enums import RPCMessageType, RunMode, SellType, State from freqtrade.exceptions import OperationalException from freqtrade.freqtradebot import FreqtradeBot from freqtrade.loggers import setup_logging from freqtrade.persistence import PairLocks, Trade -from freqtrade.rpc import RPC, RPCMessageType +from freqtrade.rpc import RPC from freqtrade.rpc.telegram import Telegram, authorized_only from tests.conftest import (create_mock_trades, get_patched_freqtradebot, log_has, patch_exchange, patch_get_signal, patch_whitelist) diff --git a/tests/rpc/test_rpc_webhook.py b/tests/rpc/test_rpc_webhook.py index 43d5dbdc5..04e63a3be 100644 --- a/tests/rpc/test_rpc_webhook.py +++ b/tests/rpc/test_rpc_webhook.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock import pytest from requests import RequestException -from freqtrade.enums import SellType -from freqtrade.rpc import RPC, RPCMessageType +from freqtrade.enums import RPCMessageType, SellType +from freqtrade.rpc import RPC from freqtrade.rpc.webhook import Webhook from tests.conftest import get_patched_freqtradebot, log_has diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 1fecb3f6a..66f87f7c9 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -11,14 +11,13 @@ import arrow import pytest from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT -from freqtrade.enums import RunMode, SellType, State +from freqtrade.enums import RPCMessageType, RunMode, SellType, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, TemporaryError) from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import Order, PairLocks, Trade from freqtrade.persistence.models import PairLock -from freqtrade.rpc import RPCMessageType from freqtrade.strategy.interface import SellCheckTuple from freqtrade.worker import Worker from tests.conftest import (create_mock_trades, get_patched_freqtradebot, get_patched_worker, From c2929260866771d4444d869f471d727a6b4169b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 10 Jun 2021 06:21:10 +0200 Subject: [PATCH 106/127] Small style improvements (no empty line at start) --- freqtrade/enums/selltype.py | 1 - freqtrade/enums/state.py | 5 ----- 2 files changed, 6 deletions(-) diff --git a/freqtrade/enums/selltype.py b/freqtrade/enums/selltype.py index e9e3dcb5a..015c30186 100644 --- a/freqtrade/enums/selltype.py +++ b/freqtrade/enums/selltype.py @@ -1,4 +1,3 @@ - from enum import Enum diff --git a/freqtrade/enums/state.py b/freqtrade/enums/state.py index d6c611ea0..572e2299f 100644 --- a/freqtrade/enums/state.py +++ b/freqtrade/enums/state.py @@ -1,8 +1,3 @@ -# pragma pylint: disable=too-few-public-methods - -""" -Bot state constant -""" from enum import Enum From c215b24a1914ead4c28688da1115c2f0b1e4c65d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Jun 2021 15:54:32 +0000 Subject: [PATCH 107/127] Bump fastapi from 0.65.1 to 0.65.2 Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.65.1 to 0.65.2. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.65.1...0.65.2) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0e4d35464..abfae55b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ python-rapidjson==1.0 sdnotify==0.3.2 # API Server -fastapi==0.65.1 +fastapi==0.65.2 uvicorn==0.14.0 pyjwt==2.1.0 aiofiles==0.7.0 From a05e38dbd340dab8500c05fcacb9b59d89baad82 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 12 Jun 2021 09:03:55 +0200 Subject: [PATCH 108/127] Require timeframe for plot-profit must be set in config or via --timeframe --- freqtrade/plot/plotting.py | 9 ++++++--- tests/test_plotting.py | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index f52b0bc81..c1b1232c2 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -47,7 +47,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0): data = load_data( datadir=config.get('datadir'), pairs=pairs, - timeframe=config.get('timeframe', '5m'), + timeframe=config['timeframe'], timerange=timerange, startup_candles=startup_candles, data_format=config.get('dataformat_ohlcv', 'json'), @@ -56,7 +56,7 @@ def init_plotscript(config, markets: List, startup_candles: int = 0): if startup_candles and data: min_date, max_date = get_timerange(data) logger.info(f"Loading data from {min_date} to {max_date}") - timerange.adjust_start_if_necessary(timeframe_to_seconds(config.get('timeframe', '5m')), + timerange.adjust_start_if_necessary(timeframe_to_seconds(config['timeframe']), startup_candles, min_date) no_trades = False @@ -583,6 +583,9 @@ def plot_profit(config: Dict[str, Any]) -> None: But should be somewhat proportional, and therefor useful in helping out to find a good algorithm. """ + if 'timeframe' not in config: + raise OperationalException('Timeframe must be set in either config or via --timeframe.') + exchange = ExchangeResolver.load_exchange(config['exchange']['name'], config) plot_elements = init_plotscript(config, list(exchange.markets)) trades = plot_elements['trades'] @@ -599,7 +602,7 @@ def plot_profit(config: Dict[str, Any]) -> None: # Create an average close price of all the pairs that were involved. # this could be useful to gauge the overall market trend fig = generate_profit_graph(plot_elements['pairs'], plot_elements['ohlcv'], - trades, config.get('timeframe', '5m'), + trades, config['timeframe'], config.get('stake_currency', '')) store_plot_file(fig, filename='freqtrade-profit-plot.html', directory=config['user_data_dir'] / 'plot', diff --git a/tests/test_plotting.py b/tests/test_plotting.py index 0b1054ced..20f159e3a 100644 --- a/tests/test_plotting.py +++ b/tests/test_plotting.py @@ -462,6 +462,10 @@ def test_plot_profit(default_conf, mocker, testdatadir): assert profit_mock.call_args_list[0][0][0] == default_conf['pairs'] assert store_mock.call_args_list[0][1]['auto_open'] is False + del default_conf['timeframe'] + with pytest.raises(OperationalException, match=r"Timeframe must be set.*--timeframe.*"): + plot_profit(default_conf) + @pytest.mark.parametrize("ind1,ind2,plot_conf,exp", [ # No indicators, use plot_conf From d35b2e3b8f545aee81742a8698d0db319210fa40 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 11:06:34 +0200 Subject: [PATCH 109/127] Update ftx stoploss logic to properly detect correct trades closes #5045 --- freqtrade/exchange/ftx.py | 13 ++++++++++++- freqtrade/freqtradebot.py | 9 +++++++-- tests/exchange/test_ftx.py | 17 ++++++++++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index 105389828..3184c2524 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -100,6 +100,17 @@ class Ftx(Exchange): order = [order for order in orders if order['id'] == order_id] if len(order) == 1: + if order[0].get('status') == 'closed': + # Trigger order was triggered ... + real_order_id = order[0].get('info', {}).get('orderId') + + order1 = self._api.fetch_order(real_order_id, pair) + # Fake type to stop - as this was really a stop order. + order1['id_stop'] = order1['id'] + order1['id'] = order_id + order1['type'] = 'stop' + order1['status_stop'] = 'triggered' + return order1 return order[0] else: raise InvalidOrderException(f"Could not get stoploss order for id {order_id}") @@ -134,5 +145,5 @@ class Ftx(Exchange): def get_order_id_conditional(self, order: Dict[str, Any]) -> str: if order['type'] == 'stop': - return safe_value_fallback2(order['info'], order, 'orderId', 'id') + return safe_value_fallback2(order, order, 'id_stop', 'id') return order['id'] diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 7aca0fcb2..a2e7fcb5d 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -816,8 +816,13 @@ class FreqtradeBot(LoggingMixin): logger.warning('Stoploss order was cancelled, but unable to recreate one.') # Finally we check if stoploss on exchange should be moved up because of trailing. - if stoploss_order and (self.config.get('trailing_stop', False) - or self.config.get('use_custom_stoploss', False)): + # Triggered Orders are now real orders - so don't replace stoploss anymore + if ( + stoploss_order + and stoploss_order.get('status_stop') != 'triggered' + and (self.config.get('trailing_stop', False) + or self.config.get('use_custom_stoploss', False)) + ): # if trailing stoploss is enabled we check if stoploss value has changed # in which case we cancel stoploss order and put another one with new # value immediately diff --git a/tests/exchange/test_ftx.py b/tests/exchange/test_ftx.py index 63d99acdf..3794bb79c 100644 --- a/tests/exchange/test_ftx.py +++ b/tests/exchange/test_ftx.py @@ -125,7 +125,7 @@ def test_stoploss_adjust_ftx(mocker, default_conf): assert not exchange.stoploss_adjust(1501, order) -def test_fetch_stoploss_order(default_conf, mocker): +def test_fetch_stoploss_order(default_conf, mocker, limit_sell_order): default_conf['dry_run'] = True order = MagicMock() order.myid = 123 @@ -147,6 +147,17 @@ def test_fetch_stoploss_order(default_conf, mocker): with pytest.raises(InvalidOrderException, match=r"Could not get stoploss order for id X"): exchange.fetch_stoploss_order('X', 'TKN/BTC')['status'] + api_mock.fetch_orders = MagicMock(return_value=[{'id': 'X', 'status': 'closed'}]) + api_mock.fetch_order = MagicMock(return_value=limit_sell_order) + + resp = exchange.fetch_stoploss_order('X', 'TKN/BTC') + assert resp + assert api_mock.fetch_order.call_count == 1 + assert resp['id_stop'] == 'mocked_limit_sell' + assert resp['id'] == 'X' + assert resp['type'] == 'stop' + assert resp['status_stop'] == 'triggered' + with pytest.raises(InvalidOrderException): api_mock.fetch_orders = MagicMock(side_effect=ccxt.InvalidOrder("Order not found")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id='ftx') @@ -165,8 +176,8 @@ def test_get_order_id(mocker, default_conf): 'type': STOPLOSS_ORDERTYPE, 'price': 1500, 'id': '1111', + 'id_stop': '1234', 'info': { - 'orderId': '1234' } } assert exchange.get_order_id_conditional(order) == '1234' @@ -175,8 +186,8 @@ def test_get_order_id(mocker, default_conf): 'type': 'limit', 'price': 1500, 'id': '1111', + 'id_stop': '1234', 'info': { - 'orderId': '1234' } } assert exchange.get_order_id_conditional(order) == '1111' From c65b4e5d3baaf2046bad848a5e061b267f767458 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 11:17:44 +0200 Subject: [PATCH 110/127] Small fix to models --- freqtrade/persistence/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/persistence/models.py b/freqtrade/persistence/models.py index 984b2de27..bbbc55c13 100644 --- a/freqtrade/persistence/models.py +++ b/freqtrade/persistence/models.py @@ -437,7 +437,7 @@ class LocalTrade(): self.close(safe_value_fallback(order, 'average', 'price')) else: raise ValueError(f'Unknown order type: {order_type}') - cleanup_db() + Trade.commit() def close(self, rate: float, *, show_msg: bool = True) -> None: """ From d54ee0eb040b975a0e71cb6ccc106babd2f55d12 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 11:24:24 +0200 Subject: [PATCH 111/127] Refactor hyperopt_tools naming --- freqtrade/commands/hyperopt_commands.py | 6 +++--- freqtrade/optimize/hyperopt.py | 4 ++-- freqtrade/optimize/hyperopt_tools.py | 4 ++-- tests/optimize/test_hyperopt.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/freqtrade/commands/hyperopt_commands.py b/freqtrade/commands/hyperopt_commands.py index dba4179ff..19337b407 100755 --- a/freqtrade/commands/hyperopt_commands.py +++ b/freqtrade/commands/hyperopt_commands.py @@ -67,7 +67,7 @@ def start_hyperopt_list(args: Dict[str, Any]) -> None: if epochs and not no_details: sorted_epochs = sorted(epochs, key=itemgetter('loss')) results = sorted_epochs[0] - HyperoptTools.print_epoch_details(results, total_epochs, print_json, no_header) + HyperoptTools.show_epoch_details(results, total_epochs, print_json, no_header) if epochs and export_csv: HyperoptTools.export_csv_file( @@ -132,8 +132,8 @@ def start_hyperopt_show(args: Dict[str, Any]) -> None: show_backtest_result(metrics['strategy_name'], metrics, metrics['stake_currency']) - HyperoptTools.print_epoch_details(val, total_epochs, print_json, no_header, - header_str="Epoch details") + HyperoptTools.show_epoch_details(val, total_epochs, print_json, no_header, + header_str="Epoch details") def hyperopt_filter_epochs(epochs: List, filteroptions: dict) -> List: diff --git a/freqtrade/optimize/hyperopt.py b/freqtrade/optimize/hyperopt.py index 49273d3de..c2b2b93cb 100644 --- a/freqtrade/optimize/hyperopt.py +++ b/freqtrade/optimize/hyperopt.py @@ -469,8 +469,8 @@ class Hyperopt: f"saved to '{self.results_file}'.") if self.current_best_epoch: - HyperoptTools.print_epoch_details(self.current_best_epoch, self.total_epochs, - self.print_json) + HyperoptTools.show_epoch_details(self.current_best_epoch, self.total_epochs, + self.print_json) else: # This is printed when Ctrl+C is pressed quickly, before first epochs have # a chance to be evaluated. diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index 6024422bc..d31b956dd 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -73,8 +73,8 @@ class HyperoptTools(): return epochs @staticmethod - def print_epoch_details(results, total_epochs: int, print_json: bool, - no_header: bool = False, header_str: str = None) -> None: + def show_epoch_details(results, total_epochs: int, print_json: bool, + no_header: bool = False, header_str: str = None) -> None: """ Display details of the hyperopt result """ diff --git a/tests/optimize/test_hyperopt.py b/tests/optimize/test_hyperopt.py index e5c05e4c0..10e99395d 100644 --- a/tests/optimize/test_hyperopt.py +++ b/tests/optimize/test_hyperopt.py @@ -1068,7 +1068,7 @@ def test_simplified_interface_failed(mocker, hyperopt_conf, method, space) -> No hyperopt.start() -def test_print_epoch_details(capsys): +def test_show_epoch_details(capsys): test_result = { 'params_details': { 'trailing': { @@ -1090,7 +1090,7 @@ def test_print_epoch_details(capsys): 'is_best': True } - HyperoptTools.print_epoch_details(test_result, 5, False, no_header=True) + HyperoptTools.show_epoch_details(test_result, 5, False, no_header=True) captured = capsys.readouterr() assert '# Trailing stop:' in captured.out # re.match(r"Pairs for .*", captured.out) From fb4dd6c2ac161fd5af1b901a0eab931d868a0673 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 11:34:44 +0200 Subject: [PATCH 112/127] Update test to cover this scenario --- tests/strategy/test_strategy_loading.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/strategy/test_strategy_loading.py b/tests/strategy/test_strategy_loading.py index 965c3d37b..bd192ecb5 100644 --- a/tests/strategy/test_strategy_loading.py +++ b/tests/strategy/test_strategy_loading.py @@ -129,13 +129,16 @@ def test_strategy_override_minimal_roi(caplog, default_conf): default_conf.update({ 'strategy': 'DefaultStrategy', 'minimal_roi': { + "20": 0.1, "0": 0.5 } }) strategy = StrategyResolver.load_strategy(default_conf) assert strategy.minimal_roi[0] == 0.5 - assert log_has("Override strategy 'minimal_roi' with value in config file: {'0': 0.5}.", caplog) + assert log_has( + "Override strategy 'minimal_roi' with value in config file: {'20': 0.1, '0': 0.5}.", + caplog) def test_strategy_override_stoploss(caplog, default_conf): From eaf0aac77e7e42767028250e8216bc8d5df706e9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 11:45:23 +0200 Subject: [PATCH 113/127] Remove OrderedDict as we're no longer supporting python 3.6 --- freqtrade/commands/list_commands.py | 3 +-- freqtrade/optimize/hyperopt_tools.py | 20 ++++---------------- freqtrade/resolvers/strategy_resolver.py | 3 +-- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/freqtrade/commands/list_commands.py b/freqtrade/commands/list_commands.py index f59adb5d8..cd26aa60e 100644 --- a/freqtrade/commands/list_commands.py +++ b/freqtrade/commands/list_commands.py @@ -1,7 +1,6 @@ import csv import logging import sys -from collections import OrderedDict from pathlib import Path from typing import Any, Dict, List @@ -154,7 +153,7 @@ def start_list_markets(args: Dict[str, Any], pairs_only: bool = False) -> None: pairs_only=pairs_only, active_only=active_only) # Sort the pairs/markets by symbol - pairs = OrderedDict(sorted(pairs.items())) + pairs = dict(sorted(pairs.items())) except Exception as e: raise OperationalException(f"Cannot get markets. Reason: {e}") from e diff --git a/freqtrade/optimize/hyperopt_tools.py b/freqtrade/optimize/hyperopt_tools.py index d31b956dd..fb1a0f56a 100644 --- a/freqtrade/optimize/hyperopt_tools.py +++ b/freqtrade/optimize/hyperopt_tools.py @@ -1,7 +1,6 @@ import io import logging -from collections import OrderedDict from pathlib import Path from typing import Any, Dict, List @@ -111,16 +110,9 @@ class HyperoptTools(): if space in ['buy', 'sell']: result_dict.setdefault('params', {}).update(space_params) elif space == 'roi': - # TODO: get rid of OrderedDict when support for python 3.6 will be - # dropped (dicts keep the order as the language feature) - # Convert keys in min_roi dict to strings because # rapidjson cannot dump dicts with integer keys... - # OrderedDict is used to keep the numeric order of the items - # in the dict. - result_dict['minimal_roi'] = OrderedDict( - (str(k), v) for k, v in space_params.items() - ) + result_dict['minimal_roi'] = {str(k): v for k, v in space_params.items()} else: # 'stoploss', 'trailing' result_dict.update(space_params) @@ -132,13 +124,9 @@ class HyperoptTools(): if space == 'stoploss': result += f"stoploss = {space_params.get('stoploss')}" elif space == 'roi': - # TODO: get rid of OrderedDict when support for python 3.6 will be - # dropped (dicts keep the order as the language feature) - minimal_roi_result = rapidjson.dumps( - OrderedDict( - (str(k), v) for k, v in space_params.items() - ), - default=str, indent=4, number_mode=rapidjson.NM_NATIVE) + minimal_roi_result = rapidjson.dumps({ + str(k): v for k, v in space_params.items() + }, default=str, indent=4, number_mode=rapidjson.NM_NATIVE) result += f"minimal_roi = {minimal_roi_result}" elif space == 'trailing': diff --git a/freqtrade/resolvers/strategy_resolver.py b/freqtrade/resolvers/strategy_resolver.py index 05fbac10d..6484f900b 100644 --- a/freqtrade/resolvers/strategy_resolver.py +++ b/freqtrade/resolvers/strategy_resolver.py @@ -6,7 +6,6 @@ This module load custom strategies import logging import tempfile from base64 import urlsafe_b64decode -from collections import OrderedDict from inspect import getfullargspec from pathlib import Path from typing import Any, Dict, Optional @@ -139,7 +138,7 @@ class StrategyResolver(IResolver): # Sort and apply type conversions if hasattr(strategy, 'minimal_roi'): - strategy.minimal_roi = OrderedDict(sorted( + strategy.minimal_roi = dict(sorted( {int(key): value for (key, value) in strategy.minimal_roi.items()}.items(), key=lambda t: t[0])) if hasattr(strategy, 'stoploss'): From cd6620a044d9ebd53584febc9ef7500e6d2dd820 Mon Sep 17 00:00:00 2001 From: Bernd Zeimetz Date: Fri, 11 Jun 2021 15:01:13 +0200 Subject: [PATCH 114/127] Ignore broken symlinks while resolving strategies. Without this fix the resolver tries to read from the broken symlink, resulting in an exception that leads to the the rather confusing error message freqtrade.resolvers.iresolver - WARNING - Path "...../user_data/strategies" does not exist. as a result of a symlink matching .py not being readable. --- freqtrade/resolvers/iresolver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index b51795e9e..5172e6fda 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -91,6 +91,9 @@ class IResolver: if not str(entry).endswith('.py'): logger.debug('Ignoring %s', entry) continue + if entry.is_symlink() and not entry.is_file(): + logger.debug('Ignoring broken symlink %s', entry) + continue module_path = entry.resolve() obj = next(cls._get_valid_object(module_path, object_name), None) From 6dc4259c6e6e664cf3bdb9f14b37447a1522152f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 05:18:54 +0000 Subject: [PATCH 115/127] Bump mkdocs-material from 7.1.7 to 7.1.8 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 7.1.7 to 7.1.8. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/docs/changelog.md) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/7.1.7...7.1.8) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 776d5592a..9b27d974b 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ mkdocs==1.2 -mkdocs-material==7.1.7 +mkdocs-material==7.1.8 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From 4530ae28cd0786611926ceb036d002f3956cdfa6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 05:19:16 +0000 Subject: [PATCH 116/127] Bump sqlalchemy from 1.4.17 to 1.4.18 Bumps [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) from 1.4.17 to 1.4.18. - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/master/CHANGES) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) --- updated-dependencies: - dependency-name: sqlalchemy dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index abfae55b9..fddf2cffe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ ccxt==1.51.3 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 -SQLAlchemy==1.4.17 +SQLAlchemy==1.4.18 python-telegram-bot==13.6 arrow==1.1.0 cachetools==4.2.2 From 3f1d6d453cbde0cceab5b1b2239bce9b5f6b1635 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 05:19:23 +0000 Subject: [PATCH 117/127] Bump mypy from 0.812 to 0.902 Bumps [mypy](https://github.com/python/mypy) from 0.812 to 0.902. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.812...v0.902) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6fbe581a5..b0f970224 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ coveralls==3.1.0 flake8==3.9.2 flake8-type-annotations==0.1.0 flake8-tidy-imports==4.3.0 -mypy==0.812 +mypy==0.902 pytest==6.2.4 pytest-asyncio==0.15.1 pytest-cov==2.12.1 From fe933e78bde88a6e3424aeafd397fbc311770b36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 05:19:33 +0000 Subject: [PATCH 118/127] Bump ccxt from 1.51.3 to 1.51.40 Bumps [ccxt](https://github.com/ccxt/ccxt) from 1.51.3 to 1.51.40. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/doc/exchanges-by-country.rst) - [Commits](https://github.com/ccxt/ccxt/compare/1.51.3...1.51.40) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index abfae55b9..781425ec5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy==1.20.3 pandas==1.2.4 -ccxt==1.51.3 +ccxt==1.51.40 # Pin cryptography for now due to rust build errors with piwheels cryptography==3.4.7 aiohttp==3.7.4.post0 From 63802aa7f66487d8a83342e757eaf69da89bc207 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jun 2021 08:52:42 +0000 Subject: [PATCH 119/127] Bump mkdocs from 1.2 to 1.2.1 Bumps [mkdocs](https://github.com/mkdocs/mkdocs) from 1.2 to 1.2.1. - [Release notes](https://github.com/mkdocs/mkdocs/releases) - [Commits](https://github.com/mkdocs/mkdocs/compare/1.2...1.2.1) --- updated-dependencies: - dependency-name: mkdocs dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 9b27d974b..352103f8b 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,4 +1,4 @@ -mkdocs==1.2 +mkdocs==1.2.1 mkdocs-material==7.1.8 mdx_truly_sane_lists==1.2 pymdown-extensions==8.2 From 4ba7a2bbd290c2c6e5746995ceacd89660a04ab5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Jun 2021 19:18:42 +0200 Subject: [PATCH 120/127] Fix mypy update problems --- freqtrade/resolvers/iresolver.py | 3 +++ freqtrade/rpc/webhook.py | 15 +++++++-------- requirements-dev.txt | 6 ++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/freqtrade/resolvers/iresolver.py b/freqtrade/resolvers/iresolver.py index 5172e6fda..5b6977b4b 100644 --- a/freqtrade/resolvers/iresolver.py +++ b/freqtrade/resolvers/iresolver.py @@ -58,6 +58,9 @@ class IResolver: # Generate spec based on absolute path # Pass object_name as first argument to have logging print a reasonable name. spec = importlib.util.spec_from_file_location(object_name or "", str(module_path)) + if not spec: + return iter([None]) + module = importlib.util.module_from_spec(spec) try: spec.loader.exec_module(module) # type: ignore # importlib does not use typehints diff --git a/freqtrade/rpc/webhook.py b/freqtrade/rpc/webhook.py index 0e4a4bf6f..b4c55649e 100644 --- a/freqtrade/rpc/webhook.py +++ b/freqtrade/rpc/webhook.py @@ -77,14 +77,13 @@ class Webhook(RPCHandler): def _send_msg(self, payload: dict) -> None: """do the actual call to the webhook""" - if self._format == 'form': - kwargs = {'data': payload} - elif self._format == 'json': - kwargs = {'json': payload} - else: - raise NotImplementedError('Unknown format: {}'.format(self._format)) - try: - post(self._url, **kwargs) + if self._format == 'form': + post(self._url, data=payload) + elif self._format == 'json': + post(self._url, json=payload) + else: + raise NotImplementedError('Unknown format: {}'.format(self._format)) + except RequestException as exc: logger.warning("Could not call webhook url. Exception: %s", exc) diff --git a/requirements-dev.txt b/requirements-dev.txt index b0f970224..924b35e1a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,3 +17,9 @@ isort==5.8.0 # Convert jupyter notebooks to markdown documents nbconvert==6.0.7 + +# mypy types +types-cachetools==0.1.7 +types-filelock==0.1.3 +types-requests==0.1.11 +types-tabulate==0.1.0 From cf7394d01cb6798213bb4b08572885328756adda Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 14 Jun 2021 19:57:24 +0200 Subject: [PATCH 121/127] Export backtesting results by default closes #4977 --- docs/backtesting.md | 21 +++++++++++++-------- freqtrade/commands/cli_options.py | 5 +++-- freqtrade/constants.py | 2 ++ freqtrade/optimize/backtesting.py | 2 +- tests/conftest.py | 1 + tests/optimize/test_backtesting.py | 12 ++++++------ tests/test_configuration.py | 5 ++--- 7 files changed, 28 insertions(+), 20 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 2027c2079..26642ef8c 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -19,7 +19,7 @@ usage: freqtrade backtesting [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--enable-protections] [--dry-run-wallet DRY_RUN_WALLET] [--strategy-list STRATEGY_LIST [STRATEGY_LIST ...]] - [--export EXPORT] [--export-filename PATH] + [--export {none,trades}] [--export-filename PATH] optional arguments: -h, --help show this help message and exit @@ -63,8 +63,8 @@ optional arguments: name is injected into the filename (so `backtest- data.json` becomes `backtest-data- DefaultStrategy.json` - --export EXPORT Export backtest results, argument are: trades. - Example: `--export=trades` + --export {none,trades} + Export backtest results (default: trades). --export-filename PATH Save backtest results to the file with this filename. Requires `--export` to be set as well. Example: @@ -100,7 +100,7 @@ Strategy arguments: Now you have good Buy and Sell strategies and some historic data, you want to test it against real data. This is what we call [backtesting](https://en.wikipedia.org/wiki/Backtesting). -Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHCLV) data from `user_data/data/` by default. +Backtesting will use the crypto-currencies (pairs) from your config file and load historical candle (OHLCV) data from `user_data/data/` by default. If no data is available for the exchange / pair / timeframe combination, backtesting will ask you to download them first using `freqtrade download-data`. For details on downloading, please refer to the [Data Downloading](data-download.md) section in the documentation. @@ -110,11 +110,16 @@ All profit calculations include fees, and freqtrade will use the exchange's defa !!! Warning "Using dynamic pairlists for backtesting" Using dynamic pairlists is possible, however it relies on the current market conditions - which will not reflect the historic status of the pairlist. - Also, when using pairlists other than StaticPairlist, reproducability of backtesting-results cannot be guaranteed. + Also, when using pairlists other than StaticPairlist, reproducibility of backtesting-results cannot be guaranteed. Please read the [pairlists documentation](plugins.md#pairlists) for more information. To achieve reproducible results, best generate a pairlist via the [`test-pairlist`](utils.md#test-pairlist) command and use that as static pairlist. +!!! Note + By default, Freqtrade will export backtesting results to `user_data/backtest_results`. + The exported trades can be used for [further analysis](#further-backtest-result-analysis) or can be used by the [plotting sub-command](plotting.md#plot-price-and-indicators) (`freqtrade plot-dataframe`) in the scripts directory. + + ### Starting balance Backtesting will require a starting balance, which can be provided as `--dry-run-wallet ` or `--starting-balance ` command line argument, or via `dry_run_wallet` configuration setting. @@ -174,13 +179,13 @@ Where `SampleStrategy1` and `AwesomeStrategy` refer to class names of strategies --- -Exporting trades to file +Prevent exporting trades to file ```bash -freqtrade backtesting --strategy backtesting --export trades --config config.json +freqtrade backtesting --strategy backtesting --export none --config config.json ``` -The exported trades can be used for [further analysis](#further-backtest-result-analysis), or can be used by the plotting script `plot_dataframe.py` in the scripts directory. +Only use this if you're sure you'll not want to plot or analyze your results further. --- diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index d832693ee..b226415e7 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -167,8 +167,9 @@ AVAILABLE_CLI_OPTIONS = { ), "export": Arg( '--export', - help='Export backtest results, argument are: trades. ' - 'Example: `--export=trades`', + help='Export backtest results (default: trades).', + choices=constants.EXPORT_OPTIONS, + ), "exportfilename": Arg( '--export-filename', diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e42b9d4b8..259aa0e03 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -12,6 +12,7 @@ PROCESS_THROTTLE_SECS = 5 # sec HYPEROPT_EPOCH = 100 # epochs RETRY_TIMEOUT = 30 # sec TIMEOUT_UNITS = ['minutes', 'seconds'] +EXPORT_OPTIONS = ['none', 'trades'] DEFAULT_DB_PROD_URL = 'sqlite:///tradesv3.sqlite' DEFAULT_DB_DRYRUN_URL = 'sqlite:///tradesv3.dryrun.sqlite' UNLIMITED_STAKE_AMOUNT = 'unlimited' @@ -308,6 +309,7 @@ CONF_SCHEMA = { 'required': ['enabled', 'listen_ip_address', 'listen_port', 'username', 'password'] }, 'db_url': {'type': 'string'}, + 'export': {'type': 'string', 'enum': EXPORT_OPTIONS, 'default': 'trades'}, 'initial_state': {'type': 'string', 'enum': ['running', 'stopped']}, 'forcebuy_enable': {'type': 'boolean'}, 'disable_dataframe_checks': {'type': 'boolean'}, diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 922f89c22..c72a8b5c5 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -520,7 +520,7 @@ class Backtesting: stats = generate_backtest_stats(data, self.all_results, min_date=min_date, max_date=max_date) - if self.config.get('export', False): + if self.config.get('export', 'none') == 'trades': store_backtest_stats(self.config['exportfilename'], stats) # Show backtest results diff --git a/tests/conftest.py b/tests/conftest.py index dd38ca610..c6a0dfcfd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -326,6 +326,7 @@ def get_default_conf(testdatadir): "strategy_path": str(Path(__file__).parent / "strategy" / "strats"), "strategy": "DefaultStrategy", "internals": {}, + "export": "none", } return configuration diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 7387c8865..60bd82d71 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -155,6 +155,7 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca 'backtesting', '--config', 'config.json', '--strategy', 'DefaultStrategy', + '--export', 'none' ] config = setup_optimize_configuration(get_args(args), RunMode.BACKTEST) @@ -172,7 +173,8 @@ def test_setup_optimize_configuration_without_arguments(mocker, default_conf, ca assert not log_has('Parameter --enable-position-stacking detected ...', caplog) assert 'timerange' not in config - assert 'export' not in config + assert 'export' in config + assert config['export'] == 'none' assert 'runmode' in config assert config['runmode'] == RunMode.BACKTEST @@ -193,7 +195,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> '--enable-position-stacking', '--disable-max-market-positions', '--timerange', ':100', - '--export', '/bar/foo', '--export-filename', 'foo_bar.json', '--fee', '0', ] @@ -223,7 +224,6 @@ def test_setup_bt_configuration_with_arguments(mocker, default_conf, caplog) -> assert log_has('Parameter --timerange detected: {} ...'.format(config['timerange']), caplog) assert 'export' in config - assert log_has('Parameter --export detected: {} ...'.format(config['export']), caplog) assert 'exportfilename' in config assert isinstance(config['exportfilename'], Path) assert log_has('Storing backtest results to {} ...'.format(config['exportfilename']), caplog) @@ -395,7 +395,7 @@ def test_backtesting_start_no_data(default_conf, mocker, caplog, testdatadir) -> default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' default_conf['timerange'] = '20180101-20180102' backtesting = Backtesting(default_conf) @@ -416,7 +416,7 @@ def test_backtesting_no_pair_left(default_conf, mocker, caplog, testdatadir) -> default_conf['timeframe'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' default_conf['timerange'] = '20180101-20180102' with pytest.raises(OperationalException, match='No pair in whitelist.'): @@ -440,7 +440,7 @@ def test_backtesting_pairlist_list(default_conf, mocker, caplog, testdatadir, ti default_conf['ticker_interval'] = "1m" default_conf['datadir'] = testdatadir - default_conf['export'] = None + default_conf['export'] = 'none' # Use stoploss from strategy del default_conf['stoploss'] default_conf['timerange'] = '20180101-20180102' diff --git a/tests/test_configuration.py b/tests/test_configuration.py index aa121edfa..c5d0cd908 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -425,7 +425,6 @@ def test_setup_configuration_without_arguments(mocker, default_conf, caplog) -> assert not log_has('Parameter --enable-position-stacking detected ...', caplog) assert 'timerange' not in config - assert 'export' not in config def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> None: @@ -448,7 +447,7 @@ def test_setup_configuration_with_arguments(mocker, default_conf, caplog) -> Non '--enable-position-stacking', '--disable-max-market-positions', '--timerange', ':100', - '--export', '/bar/foo', + '--export', 'trades', '--stake-amount', 'unlimited' ] @@ -496,7 +495,7 @@ def test_setup_configuration_with_stratlist(mocker, default_conf, caplog) -> Non 'backtesting', '--config', 'config.json', '--ticker-interval', '1m', - '--export', '/bar/foo', + '--export', 'trades', '--strategy-list', 'DefaultStrategy', 'TestStrategy' From 6d5fc967147e0aae37423d1e0ee57de47a17a791 Mon Sep 17 00:00:00 2001 From: Rokas Kupstys Date: Sat, 12 Jun 2021 10:16:30 +0300 Subject: [PATCH 122/127] Implement most pessimistic handling of trailing stoploss. --- freqtrade/optimize/backtesting.py | 16 +++++++++ freqtrade/strategy/interface.py | 4 +-- tests/optimize/test_backtest_detail.py | 47 ++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index c72a8b5c5..19ae74ae6 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -225,6 +225,22 @@ class Backtesting: # sell at open price. return sell_row[OPEN_IDX] + # Special case: trailing triggers within same candle as trade opened. Assume most + # pessimistic price movement, which is moving just enough to arm stoploss and + # immediately going down to stop price. + if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 and \ + self.strategy.trailing_stop_positive: + if self.strategy.trailing_only_offset_is_reached: + # Worst case: price reaches stop_positive_offset and dives down. + stop_rate = sell_row[OPEN_IDX] * \ + (1 + abs(self.strategy.trailing_stop_positive_offset) - + abs(self.strategy.trailing_stop_positive)) + else: + # Worst case: price ticks tiny bit above open and dives down. + stop_rate = sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive)) + assert stop_rate < sell_row[HIGH_IDX] + return stop_rate + # Set close_rate to stoploss return trade.stop_loss elif sell.sell_type == (SellType.ROI): diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 8ea38f503..47d4259fc 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -610,7 +610,7 @@ class IStrategy(ABC, HyperStrategyMixin): # Initiate stoploss with open_rate. Does nothing if stoploss is already set. trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) - if self.use_custom_stoploss: + if self.use_custom_stoploss and trade.stop_loss < current_rate: stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None )(pair=trade.pair, trade=trade, current_time=current_time, @@ -623,7 +623,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: logger.warning("CustomStoploss function did not return valid stoploss") - if self.trailing_stop: + if self.trailing_stop and trade.stop_loss < current_rate: # trailing stoploss handling sl_offset = self.trailing_stop_positive_offset diff --git a/tests/optimize/test_backtest_detail.py b/tests/optimize/test_backtest_detail.py index e5b969383..488425323 100644 --- a/tests/optimize/test_backtest_detail.py +++ b/tests/optimize/test_backtest_detail.py @@ -457,6 +457,50 @@ tc28 = BTContainer(data=[ trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=3)] ) +# Test 29: trailing_stop should be triggered by low of next candle, without adjusting stoploss using +# high of stoploss candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc29 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5050, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], # Triggers trailing-stoploss + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.02, trailing_stop=True, + trailing_stop_positive=0.03, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=2)] +) + +# Test 30: trailing_stop should be triggered immediately on trade open candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc30 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) and stop + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=-0.01, trailing_stop=True, + trailing_stop_positive=0.01, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=1)] +) + +# Test 31: trailing_stop should be triggered immediately on trade open candle. +# stop-loss: 10%, ROI: 10% (should not apply) +tc31 = BTContainer(data=[ + # D O H L C V B S + [0, 5000, 5050, 4950, 5000, 6172, 1, 0], + [1, 5000, 5500, 5000, 4900, 6172, 0, 0], # enter trade (signal on last candle) and stop + [2, 4900, 5250, 4500, 5100, 6172, 0, 0], + [3, 5100, 5100, 4650, 4750, 6172, 0, 0], + [4, 4750, 4950, 4350, 4750, 6172, 0, 0]], + stop_loss=-0.10, roi={"0": 0.10}, profit_perc=0.01, trailing_stop=True, + trailing_only_offset_is_reached=True, trailing_stop_positive_offset=0.02, + trailing_stop_positive=0.01, + trades=[BTrade(sell_reason=SellType.TRAILING_STOP_LOSS, open_tick=1, close_tick=1)] +) + TESTS = [ tc0, tc1, @@ -487,6 +531,9 @@ TESTS = [ tc26, tc27, tc28, + tc29, + tc30, + tc31, ] From 38ed49cef54d0bfe4606be46e2ec88c344e768b3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 13 Jun 2021 16:37:11 +0200 Subject: [PATCH 123/127] move low to stoploss_reached to clarify where which rate is used --- freqtrade/strategy/interface.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 47d4259fc..6358c6a4e 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -524,15 +524,14 @@ class IStrategy(ABC, HyperStrategyMixin): :param force_stoploss: Externally provided stoploss :return: True if trade should be sold, False otherwise """ - # Set current rate to low for backtesting sell - current_rate = low or rate + current_rate = rate current_profit = trade.calc_profit_ratio(current_rate) trade.adjust_min_max_rates(high or current_rate) stoplossflag = self.stop_loss_reached(current_rate=current_rate, trade=trade, current_time=date, current_profit=current_profit, - force_stoploss=force_stoploss, high=high) + force_stoploss=force_stoploss, low=low, high=high) # Set current rate to high for backtesting sell current_rate = high or rate @@ -599,18 +598,21 @@ class IStrategy(ABC, HyperStrategyMixin): def stop_loss_reached(self, current_rate: float, trade: Trade, current_time: datetime, current_profit: float, - force_stoploss: float, high: float = None) -> SellCheckTuple: + force_stoploss: float, low: float = None, + high: float = None) -> SellCheckTuple: """ Based on current profit of the trade and configured (trailing) stoploss, decides to sell or not :param current_profit: current profit as ratio + :param low: Low value of this candle, only set in backtesting + :param high: High value of this candle, only set in backtesting """ stop_loss_value = force_stoploss if force_stoploss else self.stoploss # Initiate stoploss with open_rate. Does nothing if stoploss is already set. trade.adjust_stop_loss(trade.open_rate, stop_loss_value, initial=True) - if self.use_custom_stoploss and trade.stop_loss < current_rate: + if self.use_custom_stoploss and trade.stop_loss < (low or current_rate): stop_loss_value = strategy_safe_wrapper(self.custom_stoploss, default_retval=None )(pair=trade.pair, trade=trade, current_time=current_time, @@ -623,7 +625,7 @@ class IStrategy(ABC, HyperStrategyMixin): else: logger.warning("CustomStoploss function did not return valid stoploss") - if self.trailing_stop and trade.stop_loss < current_rate: + if self.trailing_stop and trade.stop_loss < (low or current_rate): # trailing stoploss handling sl_offset = self.trailing_stop_positive_offset @@ -643,7 +645,7 @@ class IStrategy(ABC, HyperStrategyMixin): # evaluate if the stoploss was hit if stoploss is not on exchange # in Dry-Run, this handles stoploss logic as well, as the logic will not be different to # regular stoploss handling. - if ((trade.stop_loss >= current_rate) and + if ((trade.stop_loss >= (low or current_rate)) and (not self.order_types.get('stoploss_on_exchange') or self.config['dry_run'])): sell_type = SellType.STOP_LOSS @@ -652,7 +654,7 @@ class IStrategy(ABC, HyperStrategyMixin): if trade.initial_stop_loss != trade.stop_loss: sell_type = SellType.TRAILING_STOP_LOSS logger.debug( - f"{trade.pair} - HIT STOP: current price at {current_rate:.6f}, " + f"{trade.pair} - HIT STOP: current price at {(low or current_rate):.6f}, " f"stoploss is {trade.stop_loss:.6f}, " f"initial stoploss was at {trade.initial_stop_loss:.6f}, " f"trade opened at {trade.open_rate:.6f}") From 1bb04bb0c24c8a959df1fbc77617f5e1c1752d29 Mon Sep 17 00:00:00 2001 From: barbarius Date: Wed, 16 Jun 2021 11:40:55 +0200 Subject: [PATCH 124/127] Moved daily avg trade row next to total trades on backtest results --- docs/backtesting.md | 8 +++----- freqtrade/optimize/optimize_reports.py | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 26642ef8c..8e50aa356 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -284,7 +284,7 @@ A backtesting result will look like that: | Backtesting to | 2019-05-01 00:00:00 | | Max open trades | 3 | | | | -| Total trades | 429 | +| Total/Daily Avg Trades| 429 / 3.575 | | Starting balance | 0.01000000 BTC | | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | @@ -373,12 +373,11 @@ It contains some useful key metrics about performance of your strategy on backte | Backtesting to | 2019-05-01 00:00:00 | | Max open trades | 3 | | | | -| Total trades | 429 | +| Total/Daily Avg Trades| 429 / 3.575 | | Starting balance | 0.01000000 BTC | | Final balance | 0.01762792 BTC | | Absolute profit | 0.00762792 BTC | | Total profit % | 76.2% | -| Trades per day | 3.575 | | Avg. stake amount | 0.001 BTC | | Total trade volume | 0.429 BTC | | | | @@ -409,12 +408,11 @@ It contains some useful key metrics about performance of your strategy on backte - `Backtesting from` / `Backtesting to`: Backtesting range (usually defined with the `--timerange` option). - `Max open trades`: Setting of `max_open_trades` (or `--max-open-trades`) - or number of pairs in the pairlist (whatever is lower). -- `Total trades`: Identical to the total trades of the backtest output table. +- `Total/Daily Avg Trades`: Identical to the total trades of the backtest output table / Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). - `Starting balance`: Start balance - as given by dry-run-wallet (config or command line). - `Final balance`: Final balance - starting balance + absolute profit. - `Absolute profit`: Profit made in stake currency. - `Total profit %`: Total profit. Aligned to the `TOTAL` row's `Tot Profit %` from the first table. Calculated as `(End capital − Starting capital) / Starting capital`. -- `Trades per day`: Total trades divided by the backtesting duration in days (this will give you information about how many trades to expect from the strategy). - `Avg. stake amount`: Average stake amount, either `stake_amount` or the average when using dynamic stake amount. - `Total trade volume`: Volume generated on the exchange to reach the above profit. - `Best Pair` / `Worst Pair`: Best and worst performing pair, and it's corresponding `Cum Profit %`. diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 84e052ac4..64b043304 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -556,7 +556,8 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Backtesting to', strat_results['backtest_end']), ('Max open trades', strat_results['max_open_trades']), ('', ''), # Empty line to improve readability - ('Total trades', strat_results['total_trades']), + ('Total/Daily Avg Trades', + f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), ('Starting balance', round_coin_value(strat_results['starting_balance'], strat_results['stake_currency'])), ('Final balance', round_coin_value(strat_results['final_balance'], @@ -564,7 +565,6 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Absolute profit ', round_coin_value(strat_results['profit_total_abs'], strat_results['stake_currency'])), ('Total profit %', f"{round(strat_results['profit_total'] * 100, 2):}%"), - ('Trades per day', strat_results['trades_per_day']), ('Avg. stake amount', round_coin_value(strat_results['avg_stake_amount'], strat_results['stake_currency'])), ('Total trade volume', round_coin_value(strat_results['total_volume'], From 1c9def2fdbfad2e11a283b900cde6e8968145930 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 16 Jun 2021 20:17:44 +0100 Subject: [PATCH 125/127] Update freqtrade/optimize/optimize_reports.py --- freqtrade/optimize/optimize_reports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/optimize/optimize_reports.py b/freqtrade/optimize/optimize_reports.py index 64b043304..df7f721ec 100644 --- a/freqtrade/optimize/optimize_reports.py +++ b/freqtrade/optimize/optimize_reports.py @@ -556,7 +556,7 @@ def text_table_add_metrics(strat_results: Dict) -> str: ('Backtesting to', strat_results['backtest_end']), ('Max open trades', strat_results['max_open_trades']), ('', ''), # Empty line to improve readability - ('Total/Daily Avg Trades', + ('Total/Daily Avg Trades', f"{strat_results['total_trades']} / {strat_results['trades_per_day']}"), ('Starting balance', round_coin_value(strat_results['starting_balance'], strat_results['stake_currency'])), From b38ab84a13d23906b05b236f6eb9421088b9c09a Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Jun 2021 06:48:41 +0200 Subject: [PATCH 126/127] Add documentation mention about new behaviour --- docs/backtesting.md | 1 + freqtrade/optimize/backtesting.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 26642ef8c..d34381f55 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -446,6 +446,7 @@ Since backtesting lacks some detailed information about what happens within a ca - Stoploss is evaluated before ROI within one candle. So you can often see more trades with the `stoploss` sell reason comparing to the results obtained with the same strategy in the Dry Run/Live Trade modes - Low happens before high for stoploss, protecting capital first - Trailing stoploss + - Trailing Stoploss is only adjusted if it's below the candle's low (otherwise it would be triggered) - High happens first - adjusting stoploss - Low uses the adjusted stoploss (so sells with large high-low difference are backtested correctly) - ROI applies before trailing-stop, ensuring profits are "top-capped" at ROI if both ROI and trailing stop applies diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 19ae74ae6..028a9eacd 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -228,13 +228,13 @@ class Backtesting: # Special case: trailing triggers within same candle as trade opened. Assume most # pessimistic price movement, which is moving just enough to arm stoploss and # immediately going down to stop price. - if sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 and \ - self.strategy.trailing_stop_positive: + if (sell.sell_type == SellType.TRAILING_STOP_LOSS and trade_dur == 0 + and self.strategy.trailing_stop_positive): if self.strategy.trailing_only_offset_is_reached: # Worst case: price reaches stop_positive_offset and dives down. - stop_rate = sell_row[OPEN_IDX] * \ - (1 + abs(self.strategy.trailing_stop_positive_offset) - - abs(self.strategy.trailing_stop_positive)) + stop_rate = (sell_row[OPEN_IDX] * + (1 + abs(self.strategy.trailing_stop_positive_offset) - + abs(self.strategy.trailing_stop_positive))) else: # Worst case: price ticks tiny bit above open and dives down. stop_rate = sell_row[OPEN_IDX] * (1 - abs(self.strategy.trailing_stop_positive)) From a49ca9cbf78b1a0e458c3e4767344d2c71a66219 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 17 Jun 2021 06:57:35 +0200 Subject: [PATCH 127/127] Change log-level "Executing handler" msg to debug closes #5143 --- freqtrade/rpc/telegram.py | 2 +- tests/rpc/test_rpc_telegram.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index c3ddfd644..aee513017 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -56,7 +56,7 @@ def authorized_only(command_handler: Callable[..., None]) -> Callable[..., Any]: ) return wrapper - logger.info( + logger.debug( 'Executing handler: %s for chat_id: %s', command_handler.__name__, chat_id diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index bbda55c3e..d091f3837 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -2,6 +2,7 @@ # pragma pylint: disable=protected-access, unused-argument, invalid-name # pragma pylint: disable=too-many-lines, too-many-arguments +import logging import re from datetime import datetime from functools import reduce @@ -120,7 +121,7 @@ def test_cleanup(default_conf, mocker, ) -> None: def test_authorized_only(default_conf, mocker, caplog, update) -> None: patch_exchange(mocker) - + caplog.set_level(logging.DEBUG) default_conf['telegram']['enabled'] = False bot = FreqtradeBot(default_conf) rpc = RPC(bot) @@ -136,6 +137,7 @@ def test_authorized_only(default_conf, mocker, caplog, update) -> None: def test_authorized_only_unauthorized(default_conf, mocker, caplog) -> None: patch_exchange(mocker) + caplog.set_level(logging.DEBUG) chat = Chat(0xdeadbeef, 0) update = Update(randint(1, 100)) update.message = Message(randint(1, 100), datetime.utcnow(), chat)